prism 1.5.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71ec272c4385e48796979636d3d37547841cf1b27de8d61dbfc69b9300fefc5e
-  data.tar.gz: 58d2105009b3edfe9246deffaec5989ba8a63da2f68d770b80393e4970bfa89a
+  metadata.gz: 34ddf7552f6698f496fba09232eac5bcfb053d0455cdac655d81d4a9e5c60d96
+  data.tar.gz: 248141836d3f2502508f3374586bfd6590d1457dd0508494588a8905c56aea91
 SHA512:
-  metadata.gz: ad2f9f3ca25f79b5414f8144b23a529ae4cbe23e7f5fc619da54cea7676ecfc93f03bd7548824d523352765d66fe157f67fb989350c4f4b4a6f29782c796539e
-  data.tar.gz: 9e447578eb512232efe4c9263adc786701cbecbd94f2f4f6d87f2ec5f0961a5ec568acbe20aef23e810fe8f7998a8fd4f9c522cb9a9b986074f7a6e8baa4f6eb
+  metadata.gz: 6e46578f5a2dd28f487bd4ffbdc7cd34bf15f230118f53bf932bea0ccff22cc4fcd000804f9a7bc27fd3399099bbd368f9ac89b6408df7de766ad1aef24a3cda
+  data.tar.gz: 7e25c7caf0fbf963e7c7aadaf35535f3c770c1813e0b9cfd3f8d4623d2d47c5a1d8e2e487d59d75a711871b9bdf5dca92c9e85af0dd4d3f513e86d3d7a66e37d

data/CHANGELOG.md

@@ -6,6 +6,46 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [1.7.0] - 2025-12-18
+
+### Added
+
+- Support `4.1` as a version option.
+- Add `equal_loc` to `CallNode`.
+- Add `len()`/`is_empty()` to `ConstantList` and `NodeList` in the Rust API.
+
+### Changed
+
+- Rename version `3.5` to version `4.0`.
+- Fix compiling the gem from source on Windows.
+- Fix parsing of unary method calls like `42.~@`.
+- Reject `def f a, (b) = 1`.
+- Reject endless method as a block parameter default.
+- Reject variable capture in alternative pattern.
+- Many fixes in regards to memory safety, found through fuzzing.
+- Many fixes to better handle invalid syntax, also found through fuzzing.
+- Fix the ruby version used by the `ripper` translator.
+- Fix `ruby_parser` translation comment processing.
+
+## [1.6.0] - 2025-10-16
+
+### Added
+
+- Add support for passing `"current"` as the version option to `Prism.*` APIs.
+
+### Changed
+
+- Remove a compiler warning for a missing unsigned cast for a shift value.
+
+## [1.5.2] - 2025-10-09
+
+### Changed
+
+- Fix character literal forced encoding when a unicode escape sequence is used.
+- Reject `1 if foo = bar baz`.
+- Clear static literal flag on interpolated strings.
+- Reject optional argument/endless method definition ambiguity.
+
 ## [1.5.1] - 2025-09-13
 
 ### Changed
@@ -676,7 +716,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v1.5.1...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v1.7.0...HEAD
+[1.7.0]: https://github.com/ruby/prism/compare/v1.6.0...v1.7.0
+[1.6.0]: https://github.com/ruby/prism/compare/v1.5.2...v1.6.0
+[1.5.2]: https://github.com/ruby/prism/compare/v1.5.1...v1.5.2
 [1.5.1]: https://github.com/ruby/prism/compare/v1.5.0...v1.5.1
 [1.5.0]: https://github.com/ruby/prism/compare/v1.4.0...v1.5.0
 [1.4.0]: https://github.com/ruby/prism/compare/v1.3.0...v1.4.0

data/Makefile

@@ -15,6 +15,7 @@ CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion
 JAVA_WASM_CFLAGS := -g -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS)
 CC ?= cc
 AR ?= ar
+ARFLAGS ?= -r$(V0:1=v)
 WASI_SDK_PATH := /opt/wasi-sdk
 
 MAKEDIRS ?= mkdir -p
@@ -38,7 +39,7 @@ build/libprism.$(SOEXT): $(SHARED_OBJECTS)
 
 build/libprism.a: $(STATIC_OBJECTS)
 	$(ECHO) "building $@ with $(AR)"
-	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS) $(Q1:0=>/dev/null)
+	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS)
 
 javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
@@ -62,12 +63,12 @@ build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
 	$(ECHO) "building $* fuzzer"
 	$(Q) $(MAKEDIRS) $(@D)
 	$(ECHO) "building main fuzz binary"
-	$(Q) AFL_HARDEN=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
+	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 	$(ECHO) "building cmplog binary"
-	$(Q) AFL_HARDEN=1 AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
+	$(Q) AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
 
 build/fuzz.heisenbug.%: $(SOURCES) fuzz/%.c fuzz/heisenbug.c
-	$(Q) AFL_HARDEN=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
+	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 
 fuzz-debug:
 	$(ECHO) "entering debug shell"

data/README.md

@@ -116,7 +116,7 @@ Prism has been integrated into the majority of Ruby runtimes, many libraries, an
 * [JRuby](https://github.com/jruby/jruby/pull/8103) (via Java)
 * [Natalie](https://github.com/natalie-lang/natalie/pull/1213) (via C++ and Ruby)
 * [Opal](https://github.com/opal/opal/pull/2642) (via Ruby and WASM)
-* [TruffleRuby](https://github.com/oracle/truffleruby/issues/3117) (via Java)
+* [TruffleRuby](https://github.com/truffleruby/truffleruby/issues/3117) (via Java)
 
 ### Libraries
 
@@ -135,6 +135,7 @@ Prism has been integrated into the majority of Ruby runtimes, many libraries, an
 * [sorbet-eraser](https://github.com/kddnewton/sorbet-eraser/pull/25)
 * [synvert](https://github.com/xinminlabs/synvert-core-ruby)
 * [typeprof](https://github.com/ruby/typeprof)
+* [unparser](https://github.com/mbj/unparser) (via parser translator)
 
 ### Applications
 

data/config.yml

@@ -60,6 +60,7 @@ errors:
   - CONDITIONAL_WHILE_PREDICATE
   - CONSTANT_PATH_COLON_COLON_CONSTANT
   - DEF_ENDLESS
+  - DEF_ENDLESS_PARAMETERS
   - DEF_ENDLESS_SETTER
   - DEF_NAME
   - DEF_PARAMS_TERM
@@ -218,6 +219,7 @@ errors:
   - PARAMETER_WILD_LOOSE_COMMA
   - PATTERN_ARRAY_MULTIPLE_RESTS
   - PATTERN_CAPTURE_DUPLICATE
+  - PATTERN_CAPTURE_IN_ALTERNATIVE
   - PATTERN_EXPRESSION_AFTER_BRACKET
   - PATTERN_EXPRESSION_AFTER_COMMA
   - PATTERN_EXPRESSION_AFTER_HROCKET
@@ -279,6 +281,7 @@ errors:
   - UNEXPECTED_INDEX_KEYWORDS
   - UNEXPECTED_LABEL
   - UNEXPECTED_MULTI_WRITE
+  - UNEXPECTED_PARAMETER_DEFAULT_VALUE
   - UNEXPECTED_RANGE_OPERATOR
   - UNEXPECTED_SAFE_NAVIGATION
   - UNEXPECTED_TOKEN_CLOSE_CONTEXT
@@ -355,6 +358,8 @@ tokens:
     comment: "a newline character outside of other tokens"
   - name: PARENTHESIS_RIGHT
     comment: ")"
+  - name: PIPE
+    comment: "|"
   - name: SEMICOLON
     comment: ";"
   # Tokens from here on are not used for lookup, and can be in any order.
@@ -588,8 +593,6 @@ tokens:
     comment: "%I"
   - name: PERCENT_UPPER_W
     comment: "%W"
-  - name: PIPE
-    comment: "|"
   - name: PIPE_EQUAL
     comment: "|="
   - name: PIPE_PIPE
@@ -1514,6 +1517,16 @@ nodes:
 
               foo(bar)
                      ^
+      - name: equal_loc
+        type: location?
+        comment: |
+          Represents the location of the equal sign, in the case that this is an attribute write.
+
+              foo.bar = value
+                      ^
+
+              foo[bar] = value
+                       ^
       - name: block
         type: node?
         kind:
@@ -1800,7 +1813,7 @@ nodes:
           Represents the predicate of the case statement. This can be either `nil` or any [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
 
               case true; when false; end
-              ^^^^
+                   ^^^^
       - name: conditions
         type: node[]
         kind: WhenNode
@@ -2615,11 +2628,18 @@ nodes:
       - name: block
         type: node?
         kind: BlockNode
+        comment: |
+          All other arguments are forwarded as normal, except the original block is replaced with the new block.
     comment: |
-      Represents the use of the `super` keyword without parentheses or arguments.
+      Represents the use of the `super` keyword without parentheses or arguments, but which might have a block.
 
           super
           ^^^^^
+
+          super { 123 }
+          ^^^^^^^^^^^^^
+
+      If it has any other arguments, it would be a `SuperNode` instead.
   - name: GlobalVariableAndWriteNode
     fields:
       - name: name
@@ -3286,6 +3306,9 @@ nodes:
           - EmbeddedVariableNode
           - InterpolatedStringNode # `"a" "#{b}"`
           - on error: XStringNode # `<<`FOO` "bar"
+          - on error: InterpolatedXStringNode
+          - on error: SymbolNode
+          - on error: InterpolatedSymbolNode
       - name: closing_loc
         type: location?
     newline: parts
@@ -4495,6 +4518,7 @@ nodes:
       - name: arguments
         type: node?
         kind: ArgumentsNode
+        comment: "Can be only `nil` when there are empty parentheses, like `super()`."
       - name: rparen_loc
         type: location?
       - name: block
@@ -4510,6 +4534,8 @@ nodes:
 
           super foo, bar
           ^^^^^^^^^^^^^^
+
+      If no arguments are provided (except for a block), it would be a `ForwardingSuperNode` instead.
   - name: SymbolNode
     flags: SymbolFlags
     fields:

data/docs/build_system.md

@@ -58,12 +58,12 @@ prism's `Makefile` is not used at all in CRuby. Instead, CRuby's `Makefile` is u
 
 ### Building prism as part of TruffleRuby
 
-[This script](https://github.com/oracle/truffleruby/blob/master/tool/import-prism.sh) imports prism sources in TruffleRuby.
+[This script](https://github.com/truffleruby/truffleruby/blob/master/tool/import-prism.sh) imports prism sources in TruffleRuby.
 The script generates the templates when importing.
 
 Then when `mx build` builds TruffleRuby and the `prism` mx project inside, it runs `make`.
 
-Then the `prism bindings` mx project is built, which contains the [bindings](https://github.com/oracle/truffleruby/blob/vm-24.1.1/src/main/c/yarp_bindings/src/yarp_bindings.c)
+Then the `prism bindings` mx project is built, which contains the [bindings](https://github.com/truffleruby/truffleruby/blob/vm-24.1.1/src/main/c/yarp_bindings/src/yarp_bindings.c)
 and links to `libprism.a` (to avoid exporting symbols, so no conflict when installing the prism gem).
 
 ### Building prism as part of JRuby

data/docs/cruby_compilation.md

@@ -1,6 +1,6 @@
 # Compiling Prism's AST
 
-One important class of consumers of Prism's AST is compilers. Currently [CRuby](https://github.com/ruby/ruby), [JRuby](https://github.com/jruby/jruby), [TruffleRuby](https://github.com/oracle/truffleruby), and [Natalie](https://github.com/natalie-lang/natalie) have all built compilation code on top of Prism's AST.
+One important class of consumers of Prism's AST is compilers. Currently [CRuby](https://github.com/ruby/ruby), [JRuby](https://github.com/jruby/jruby), [TruffleRuby](https://github.com/truffleruby/truffleruby), and [Natalie](https://github.com/natalie-lang/natalie) have all built compilation code on top of Prism's AST.
 
 This document will describe, at a high level, how CRuby's compilation of Prism's AST works.
 

data/docs/design.md

@@ -18,11 +18,11 @@ The templated files contain all of the code required to allocate and initialize
 
 In order to provide the best possible error tolerance, the parser is hand-written. It is structured using Pratt parsing, a technique developed by Vaughan Pratt back in the 1970s. Below are a bunch of links to articles and papers that explain Pratt parsing in more detail.
 
-* https://web.archive.org/web/20151223215421/http://hall.org.ua/halls/wizzard/pdf/Vaughan.Pratt.TDOP.pdf
+* https://github.com/tdop/tdop.github.io/raw/master/original.pdf
 * https://tdop.github.io/
 * https://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/
 * https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html
-* https://chidiwilliams.com/post/on-recursive-descent-and-pratt-parsing/
+* https://chidiwilliams.com/posts/on-recursive-descent-and-pratt-parsing
 
 You can find most of the functions that correspond to constructs in the Pratt parsing algorithm in `prism.c`. As a couple of examples:
 

data/docs/parser_translation.md

@@ -20,5 +20,5 @@ Prism::Translation::ParserCurrent.parse("puts 'Hello World!'")
 
 All the parsers are autoloaded, so you don't have to worry about requiring them yourself.
 
-If you also need to parse Ruby versions below 3.3 (which `prism` has no support for), check out
+If you also need to parse Ruby versions below 3.3 (for which the `prism` translation layer does not have explicit support), check out
 [this guide](https://github.com/whitequark/parser/blob/master/doc/PRISM_TRANSLATION.md) from the `parser` gem on how to use both in conjunction.

data/docs/releasing.md

@@ -40,14 +40,17 @@ ruby -pi -e 'gsub(/^ruby-prism-sys = \{ version = ".+?"/, %Q{ruby-prism-sys = \{
 * Update the `Gemfile.lock` file:
 
 ```sh
-chruby ruby-3.5.0-dev
+chruby ruby-4.0.0-dev
 bundle install
 ```
 
 * Update the version-specific lockfiles:
 
 ```sh
-bin/prism bundle install
+for VERSION in "2.7" "3.0" "3.1" "3.2" "3.3" "3.4"; do docker run -it --rm -v "$PWD":/usr/src/app -w /usr/src/app -e BUNDLE_GEMFILE="gemfiles/$VERSION/Gemfile" "ruby:$VERSION" bundle update; done
+docker run -it --rm -v "$PWD":/usr/src/app -w /usr/src/app -e BUNDLE_GEMFILE="gemfiles/4.0/Gemfile" ruby:4.0.0-preview2 bundle update
+docker run -it --rm -v "$PWD":/usr/src/app -w /usr/src/app -e BUNDLE_GEMFILE="gemfiles/jruby/Gemfile" jruby:latest bundle update
+BUNDLE_GEMFILE=gemfiles/truffleruby/Gemfile chruby-exec truffleruby -- bundle update
 ```
 
 * Update the cargo lockfiles:
@@ -71,26 +74,4 @@ git push
 ## Publishing
 
 * Update the GitHub release page with a copy of the latest entry in the `CHANGELOG.md` file.
-* Publish the gem to [rubygems.org](rubygems.org). Note that you must have access to the `prism` gem to do this.
-
-```sh
-bundle exec rake release
-```
-
-* Generate the `wasm` artifact (or download it from GitHub actions and put it in `javascript/src/prism.wasm`).
-
-```sh
-make wasm
-```
-
-* Publish the JavaScript package to [npmjs.com](npmjs.com). Note that you must have access to the `@ruby/prism` package to do this.
-
-```sh
-npm publish
-```
-
-* Publish the rust crate to [crates.io](crates.io). Note that you must have access to the `ruby-prism-sys` and `ruby-prism` crates to do this.
-
-```sh
-bundle exec rake cargo:publish:real
-```
+* Push a new tag to the GitHub repository, following the `vX.Y.Z` format.

data/ext/prism/api_node.c

@@ -1789,7 +1789,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 190 "prism/templates/ext/prism/api_node.c.erb"
                 case PM_CALL_NODE: {
                     pm_call_node_t *cast = (pm_call_node_t *) node;
-                    VALUE argv[12];
+                    VALUE argv[13];
 
                     // source
                     argv[0] = source;
@@ -1832,11 +1832,15 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 243 "prism/templates/ext/prism/api_node.c.erb"
                     argv[10] = cast->closing_loc.start == NULL ? Qnil : pm_location_new(parser, cast->closing_loc.start, cast->closing_loc.end, source, freeze);
 
+                    // equal_loc
+#line 243 "prism/templates/ext/prism/api_node.c.erb"
+                    argv[11] = cast->equal_loc.start == NULL ? Qnil : pm_location_new(parser, cast->equal_loc.start, cast->equal_loc.end, source, freeze);
+
                     // block
 #line 213 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[11] = rb_ary_pop(value_stack);
+                    argv[12] = rb_ary_pop(value_stack);
 
-                    VALUE value = rb_class_new_instance(12, argv, rb_cPrismCallNode);
+                    VALUE value = rb_class_new_instance(13, argv, rb_cPrismCallNode);
                     if (freeze) rb_obj_freeze(value);
 
                     rb_ary_push(value_stack, value);

data/ext/prism/extconf.rb

@@ -37,7 +37,7 @@ end
 def generate_templates
   Dir.chdir(File.expand_path("../..", __dir__)) do
     if !File.exist?("include/prism/ast.h") && Dir.exist?(".git")
-      system("templates/template.rb", exception: true)
+      system(RbConfig.ruby, "templates/template.rb", exception: true)
     end
   end
 end

data/ext/prism/extension.c

@@ -25,6 +25,7 @@ VALUE rb_cPrismLexResult;
 VALUE rb_cPrismParseLexResult;
 VALUE rb_cPrismStringQuery;
 VALUE rb_cPrismScope;
+VALUE rb_cPrismCurrentVersionError;
 
 VALUE rb_cPrismDebugEncoding;
 
@@ -199,7 +200,12 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
         if (!NIL_P(value)) {
             const char *version = check_string(value);
 
-            if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
+            if (RSTRING_LEN(value) == 7 && strncmp(version, "current", 7) == 0) {
+                const char *current_version = RSTRING_PTR(rb_const_get(rb_cObject, rb_intern("RUBY_VERSION")));
+                if (!pm_options_version_set(options, current_version, 3)) {
+                    rb_exc_raise(rb_exc_new_cstr(rb_cPrismCurrentVersionError, current_version));
+                }
+            } else if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
                 rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value);
             }
         }
@@ -888,7 +894,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  *       version of Ruby syntax (which you can trigger with `nil` or
  *       `"latest"`). You may also restrict the syntax to a specific version of
  *       Ruby, e.g., with `"3.3.0"`. To parse with the same syntax version that
- *       the current Ruby is running use `version: RUBY_VERSION`. Raises
+ *       the current Ruby is running use `version: "current"`. Raises
  *       ArgumentError if the version is not currently supported by Prism.
  */
 static VALUE
@@ -1364,6 +1370,8 @@ Init_prism(void) {
     rb_cPrismStringQuery = rb_define_class_under(rb_cPrism, "StringQuery", rb_cObject);
     rb_cPrismScope = rb_define_class_under(rb_cPrism, "Scope", rb_cObject);
 
+    rb_cPrismCurrentVersionError = rb_const_get(rb_cPrism, rb_intern("CurrentVersionError"));
+
     // Intern all of the IDs eagerly that we support so that we don't have to do
     // it every time we parse.
     rb_id_option_command_line = rb_intern_const("command_line");

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "1.5.1"
+#define EXPECTED_PRISM_VERSION "1.7.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>