prism 1.4.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0209a58426470fc6afedb5922c9bf9264cb9ccd2debbef68fb1a307a4d1be78
-  data.tar.gz: 491a11ab3d9d840f4ac228e8b79e85ddb24ae1f82b3cee7cbdb1003bea9f33f2
+  metadata.gz: 2ef507799374ba0281ff3d8ab78fc29f673b3fc213f695ae68c65cf0d38b4bcc
+  data.tar.gz: 7610a756cffd1aeb517f135446ae25360568cc00e6d78e748470469bd345463c
 SHA512:
-  metadata.gz: 30b9cb40262daf924f047034c6314b418c367691eccc188acd08328bd92e6a51d3818a91c3d17fb5dc3f159129186ef6c4bd7bfbe815b9c8f268fa9b938fd1ac
-  data.tar.gz: a1b9ab207e98c779b0d371f2f14e6c5fb8315273af7518bc8e8135ba3dff4f1385787e169a4e4c146aa1ea7c10f6fce4f82d37958f88e8f68f1e8dabe54115bf
+  metadata.gz: 539d064482f82c7fd85c7e8269b1c400fd576b20effbcb857636667ce8818367eea35446bc7ee2decf3cdde7171883e8aeaa154c2bf1b8d7f3c7fa5c12203f08
+  data.tar.gz: 59181adcac710bf7697b1dec0556eb306f9cf79d2b7492e0392d9698401dd3abb5a1406fc5d66d1c4d0acbd85d9ebf8e674683e9ec585d765216953f80a75297

data/CHANGELOG.md

@@ -6,6 +6,52 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [1.6.0] - 2025-10-16
+
+### Added
+
+- Add support for passing `"current"` as the version option to `Prism.*` APIs.
+
+### Changed
+
+- Remove a compiler warning for a missing unsigned cast for a shift value.
+
+## [1.5.2] - 2025-10-09
+
+### Changed
+
+- Fix character literal forced encoding when a unicode escape sequence is used.
+- Reject `1 if foo = bar baz`.
+- Clear static literal flag on interpolated strings.
+- Reject optional argument/endless method definition ambiguity.
+
+## [1.5.1] - 2025-09-13
+
+### Changed
+
+- Revert of a bug introduced with static literal flags on interpolated strings.
+
+## [1.5.0] - 2025-09-12
+
+### Added
+
+- Add `Prism::Translation::ParserCurrent`.
+- Add `Integer::to_u32_digits` for the Rust API.
+- Add `pm_comment_type_t` field for the Rust API.
+- Support leading logical operators for CRuby 3.5+.
+
+### Changed
+
+- Mark Prism as ractor-safe.
+- Enforce a minimum version for the parser translation layer.
+- Many fixes to the parser translation layer.
+- Accept a newline after the `defined?` keyword.
+- Reject `true && not true`.
+- Make `it = it` assign nil to match parse.y behavior [Bug #21139].
+- Some fixes to the ruby parser translation layer.
+- Ensure call nodes have the correct ending location.
+- Reject `foo && return bar`.
+
 ## [1.4.0] - 2025-03-18
 
 ### Added
@@ -649,7 +695,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v1.4.0...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v1.6.0...HEAD
+[1.6.0]: https://github.com/ruby/prism/compare/v1.5.2...v1.6.0
+[1.5.2]: https://github.com/ruby/prism/compare/v1.5.1...v1.5.2
+[1.5.1]: https://github.com/ruby/prism/compare/v1.5.0...v1.5.1
+[1.5.0]: https://github.com/ruby/prism/compare/v1.4.0...v1.5.0
 [1.4.0]: https://github.com/ruby/prism/compare/v1.3.0...v1.4.0
 [1.3.0]: https://github.com/ruby/prism/compare/v1.2.0...v1.3.0
 [1.2.0]: https://github.com/ruby/prism/compare/v1.1.0...v1.2.0

data/Makefile

@@ -12,8 +12,10 @@ SOEXT ?= $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
 CPPFLAGS := -Iinclude $(CPPFLAGS)
 CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(CFLAGS)
+JAVA_WASM_CFLAGS := -g -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS)
 CC ?= cc
 AR ?= ar
+ARFLAGS ?= -r$(V0:1=v)
 WASI_SDK_PATH := /opt/wasi-sdk
 
 MAKEDIRS ?= mkdir -p
@@ -37,7 +39,7 @@ build/libprism.$(SOEXT): $(SHARED_OBJECTS)
 
 build/libprism.a: $(STATIC_OBJECTS)
 	$(ECHO) "building $@ with $(AR)"
-	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS) $(Q1:0=>/dev/null)
+	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS)
 
 javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
@@ -45,7 +47,7 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 
 java-wasm/src/test/resources/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
-	$(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES)
+	$(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES)
 
 build/shared/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"

data/README.md

@@ -135,7 +135,9 @@ Prism has been integrated into the majority of Ruby runtimes, many libraries, an
 * [sorbet-eraser](https://github.com/kddnewton/sorbet-eraser/pull/25)
 * [synvert](https://github.com/xinminlabs/synvert-core-ruby)
 * [typeprof](https://github.com/ruby/typeprof)
+* [unparser](https://github.com/mbj/unparser) (via parser translator)
 
 ### Applications
 
 * [gem.sh](https://github.com/marcoroth/gem.sh/pull/96)
+* [Sorbet](https://github.com/sorbet/sorbet)

data/config.yml

@@ -60,6 +60,7 @@ errors:
   - CONDITIONAL_WHILE_PREDICATE
   - CONSTANT_PATH_COLON_COLON_CONSTANT
   - DEF_ENDLESS
+  - DEF_ENDLESS_PARAMETERS
   - DEF_ENDLESS_SETTER
   - DEF_NAME
   - DEF_PARAMS_TERM
@@ -101,6 +102,8 @@ errors:
   - EXPECT_FOR_DELIMITER
   - EXPECT_IDENT_REQ_PARAMETER
   - EXPECT_IN_DELIMITER
+  - EXPECT_LPAREN_AFTER_NOT_LPAREN
+  - EXPECT_LPAREN_AFTER_NOT_OTHER
   - EXPECT_LPAREN_REQ_PARAMETER
   - EXPECT_MESSAGE
   - EXPECT_RBRACKET
@@ -320,13 +323,42 @@ warnings:
   - UNUSED_LOCAL_VARIABLE
   - VOID_STATEMENT
 tokens:
+  # The order of the tokens at the beginning is important, because we use them
+  # for a lookup table.
   - name: EOF
     value: 1
     comment: final token in the file
-  - name: MISSING
-    comment: "a token that was expected but not found"
-  - name: NOT_PROVIDED
-    comment: "a token that was not present but it is okay"
+  - name: BRACE_RIGHT
+    comment: "}"
+  - name: COMMA
+    comment: ","
+  - name: EMBEXPR_END
+    comment: "}"
+  - name: KEYWORD_DO
+    comment: "do"
+  - name: KEYWORD_ELSE
+    comment: "else"
+  - name: KEYWORD_ELSIF
+    comment: "elsif"
+  - name: KEYWORD_END
+    comment: "end"
+  - name: KEYWORD_ENSURE
+    comment: "ensure"
+  - name: KEYWORD_IN
+    comment: "in"
+  - name: KEYWORD_RESCUE
+    comment: "rescue"
+  - name: KEYWORD_THEN
+    comment: "then"
+  - name: KEYWORD_WHEN
+    comment: "when"
+  - name: NEWLINE
+    comment: "a newline character outside of other tokens"
+  - name: PARENTHESIS_RIGHT
+    comment: ")"
+  - name: SEMICOLON
+    comment: ";"
+  # Tokens from here on are not used for lookup, and can be in any order.
   - name: AMPERSAND
     comment: "&"
   - name: AMPERSAND_AMPERSAND
@@ -349,8 +381,6 @@ tokens:
     comment: "!~"
   - name: BRACE_LEFT
     comment: "{"
-  - name: BRACE_RIGHT
-    comment: "}"
   - name: BRACKET_LEFT
     comment: "["
   - name: BRACKET_LEFT_ARRAY
@@ -373,8 +403,6 @@ tokens:
     comment: ":"
   - name: COLON_COLON
     comment: "::"
-  - name: COMMA
-    comment: ","
   - name: COMMENT
     comment: "a comment"
   - name: CONSTANT
@@ -393,8 +421,6 @@ tokens:
     comment: "a line inside of embedded documentation"
   - name: EMBEXPR_BEGIN
     comment: "#{"
-  - name: EMBEXPR_END
-    comment: "}"
   - name: EMBVAR
     comment: "#"
   - name: EQUAL
@@ -461,20 +487,10 @@ tokens:
     comment: "def"
   - name: KEYWORD_DEFINED
     comment: "defined?"
-  - name: KEYWORD_DO
-    comment: "do"
   - name: KEYWORD_DO_LOOP
     comment: "do keyword for a predicate in a while, until, or for loop"
-  - name: KEYWORD_ELSE
-    comment: "else"
-  - name: KEYWORD_ELSIF
-    comment: "elsif"
-  - name: KEYWORD_END
-    comment: "end"
   - name: KEYWORD_END_UPCASE
     comment: "END"
-  - name: KEYWORD_ENSURE
-    comment: "ensure"
   - name: KEYWORD_FALSE
     comment: "false"
   - name: KEYWORD_FOR
@@ -483,8 +499,6 @@ tokens:
     comment: "if"
   - name: KEYWORD_IF_MODIFIER
     comment: "if in the modifier form"
-  - name: KEYWORD_IN
-    comment: "in"
   - name: KEYWORD_MODULE
     comment: "module"
   - name: KEYWORD_NEXT
@@ -497,8 +511,6 @@ tokens:
     comment: "or"
   - name: KEYWORD_REDO
     comment: "redo"
-  - name: KEYWORD_RESCUE
-    comment: "rescue"
   - name: KEYWORD_RESCUE_MODIFIER
     comment: "rescue in the modifier form"
   - name: KEYWORD_RETRY
@@ -509,8 +521,6 @@ tokens:
     comment: "self"
   - name: KEYWORD_SUPER
     comment: "super"
-  - name: KEYWORD_THEN
-    comment: "then"
   - name: KEYWORD_TRUE
     comment: "true"
   - name: KEYWORD_UNDEF
@@ -523,8 +533,6 @@ tokens:
     comment: "until"
   - name: KEYWORD_UNTIL_MODIFIER
     comment: "until in the modifier form"
-  - name: KEYWORD_WHEN
-    comment: "when"
   - name: KEYWORD_WHILE
     comment: "while"
   - name: KEYWORD_WHILE_MODIFIER
@@ -561,16 +569,12 @@ tokens:
     comment: "-="
   - name: MINUS_GREATER
     comment: "->"
-  - name: NEWLINE
-    comment: "a newline character outside of other tokens"
   - name: NUMBERED_REFERENCE
     comment: "a numbered reference to a capture group in the previous regular expression match"
   - name: PARENTHESIS_LEFT
     comment: "("
   - name: PARENTHESIS_LEFT_PARENTHESES
     comment: "( for a parentheses node"
-  - name: PARENTHESIS_RIGHT
-    comment: ")"
   - name: PERCENT
     comment: "%"
   - name: PERCENT_EQUAL
@@ -603,8 +607,6 @@ tokens:
     comment: "the beginning of a regular expression"
   - name: REGEXP_END
     comment: "the end of a regular expression"
-  - name: SEMICOLON
-    comment: ";"
   - name: SLASH
     comment: "/"
   - name: SLASH_EQUAL
@@ -649,6 +651,10 @@ tokens:
     comment: "a separator between words in a list"
   - name: __END__
     comment: "marker for the point in the file at which the parser should stop"
+  - name: MISSING
+    comment: "a token that was expected but not found"
+  - name: NOT_PROVIDED
+    comment: "a token that was not present but it is okay"
 flags:
   - name: ArgumentsNodeFlags
     values:
@@ -987,8 +993,19 @@ nodes:
       - name: constant
         type: node?
         kind:
-          - ConstantReadNode
           - ConstantPathNode
+          - ConstantReadNode
+        comment: |
+          Represents the optional constant preceding the Array
+
+              foo in Bar[]
+                     ^^^
+
+              foo in Bar[1, 2, 3]
+                     ^^^
+
+              foo in Bar::Baz[1, 2, 3]
+                     ^^^^^^^^
       - name: requireds
         type: node[]
         kind: pattern expression
@@ -1784,7 +1801,7 @@ nodes:
           Represents the predicate of the case statement. This can be either `nil` or any [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
 
               case true; when false; end
-              ^^^^
+                   ^^^^
       - name: conditions
         type: node[]
         kind: WhenNode
@@ -1828,6 +1845,11 @@ nodes:
         type: constant[]
       - name: class_keyword_loc
         type: location
+        comment: |
+          Represents the location of the `class` keyword.
+
+              class Foo end
+              ^^^^^
       - name: constant_path
         type: node
         kind:
@@ -1836,18 +1858,43 @@ nodes:
           - on error: CallNode # class 0.X end
       - name: inheritance_operator_loc
         type: location?
+        comment: |
+          Represents the location of the `<` operator.
+
+              class Foo < Bar
+                        ^
       - name: superclass
         type: node?
         kind: non-void expression
+        comment: |
+          Represents the superclass of the class.
+
+              class Foo < Bar
+                          ^^^
       - name: body
         type: node?
         kind:
           - StatementsNode
           - BeginNode
+        comment: |
+          Represents the body of the class.
+
+              class Foo
+                foo
+                ^^^
       - name: end_keyword_loc
         type: location
+        comment: |
+          Represents the location of the `end` keyword.
+
+              class Foo end
+                        ^^^
       - name: name
         type: constant
+        comment: |
+          The name of the class.
+
+              class Foo end # name `:Foo`
     comment: |
       Represents a class declaration involving the `class` keyword.
 
@@ -2374,23 +2421,68 @@ nodes:
       - name: constant
         type: node?
         kind:
-          - ConstantReadNode
           - ConstantPathNode
+          - ConstantReadNode
+        comment: |
+          Represents the optional constant preceding the pattern
+
+              foo in Foo(*bar, baz, *qux)
+                     ^^^
       - name: left
         type: node
         kind: SplatNode
+        comment: |
+          Represents the first wildcard node in the pattern.
+
+              foo in *bar, baz, *qux
+                     ^^^^
+
+              foo in Foo(*bar, baz, *qux)
+                         ^^^^
       - name: requireds
         type: node[]
         kind: pattern expression
+        comment: |
+          Represents the nodes in between the wildcards.
+
+              foo in *bar, baz, *qux
+                           ^^^
+
+              foo in Foo(*bar, baz, 1, *qux)
+                               ^^^^^^
       - name: right
         type: node
         kind:
           - SplatNode
           - on error: MissingNode
+        comment: |
+          Represents the second wildcard node in the pattern.
+
+              foo in *bar, baz, *qux
+                                ^^^^
+
+              foo in Foo(*bar, baz, *qux)
+                                    ^^^^
       - name: opening_loc
         type: location?
+        comment: |
+          The location of the opening brace.
+
+              foo in [*bar, baz, *qux]
+                     ^
+
+              foo in Foo(*bar, baz, *qux)
+                        ^
       - name: closing_loc
         type: location?
+        comment: |
+          The location of the closing brace.
+
+              foo in [*bar, baz, *qux]
+                                     ^
+
+              foo in Foo(*bar, baz, *qux)
+                                        ^
     comment: |
       Represents a find pattern in pattern matching.
 
@@ -2402,6 +2494,9 @@ nodes:
 
           foo in Foo(*bar, baz, *qux)
                  ^^^^^^^^^^^^^^^^^^^^
+
+          foo => *bar, baz, *qux
+                 ^^^^^^^^^^^^^^^
   - name: FlipFlopNode
     flags: RangeFlags
     fields:
@@ -2679,20 +2774,60 @@ nodes:
       - name: constant
         type: node?
         kind:
-          - ConstantReadNode
           - ConstantPathNode
+          - ConstantReadNode
+        comment: |
+          Represents the optional constant preceding the Hash.
+
+              foo => Bar[a: 1, b: 2]
+                   ^^^
+
+              foo => Bar::Baz[a: 1, b: 2]
+                   ^^^^^^^^
       - name: elements
         type: node[]
         kind: AssocNode
+        comment: |
+          Represents the explicit named hash keys and values.
+
+              foo => { a: 1, b:, ** }
+                       ^^^^^^^^
       - name: rest
         type: node?
         kind:
           - AssocSplatNode
           - NoKeywordsParameterNode
+        comment: |
+          Represents the rest of the Hash keys and values. This can be named, unnamed, or explicitly forbidden via `**nil`, this last one results in a `NoKeywordsParameterNode`.
+
+              foo => { a: 1, b:, **c }
+                                 ^^^
+
+              foo => { a: 1, b:, ** }
+                                 ^^
+
+              foo => { a: 1, b:, **nil }
+                                 ^^^^^
       - name: opening_loc
         type: location?
+        comment: |
+          The location of the opening brace.
+
+              foo => { a: 1 }
+                     ^
+
+              foo => Bar[a: 1]
+                        ^
       - name: closing_loc
         type: location?
+        comment: |
+          The location of the closing brace.
+
+              foo => { a: 1 }
+                            ^
+
+              foo => Bar[a: 1]
+                             ^
     comment: |
       Represents a hash pattern in pattern matching.
 
@@ -2701,6 +2836,12 @@ nodes:
 
           foo => { a: 1, b: 2, **c }
                  ^^^^^^^^^^^^^^^^^^^
+
+          foo => Bar[a: 1, b: 2]
+                 ^^^^^^^^^^^^^^^
+
+          foo in { a: 1, b: 2 }
+                 ^^^^^^^^^^^^^^
   - name: IfNode
     fields:
       - name: if_keyword_loc
@@ -3353,6 +3494,9 @@ nodes:
 
           foo, bar = baz
           ^^^  ^^^
+
+          foo => baz
+                 ^^^
   - name: LocalVariableWriteNode
     fields:
       - name: name
@@ -3443,11 +3587,65 @@ nodes:
       - name: value
         type: node
         kind: non-void expression
+        comment: |
+          Represents the left-hand side of the operator.
+
+              foo => bar
+              ^^^
       - name: pattern
         type: node
         kind: pattern expression
+        comment: |
+          Represents the right-hand side of the operator. The type of the node depends on the expression.
+
+          Anything that looks like a local variable name (including `_`) will result in a `LocalVariableTargetNode`.
+
+              foo => a # This is equivalent to writing `a = foo`
+                     ^
+
+          Using an explicit `Array` or combining expressions with `,` will result in a `ArrayPatternNode`. This can be preceded by a constant.
+
+              foo => [a]
+                     ^^^
+
+              foo => a, b
+                     ^^^^
+
+              foo => Bar[a, b]
+                     ^^^^^^^^^
+
+          If the array pattern contains at least two wildcard matches, a `FindPatternNode` is created instead.
+
+              foo => *, 1, *a
+                     ^^^^^
+
+          Using an explicit `Hash` or a constant with square brackets and hash keys in the square brackets will result in a `HashPatternNode`.
+
+              foo => { a: 1, b: }
+
+              foo => Bar[a: 1, b:]
+
+              foo => Bar[**]
+
+          To use any variable that needs run time evaluation, pinning is required. This results in a `PinnedVariableNode`
+
+              foo => ^a
+                     ^^
+
+          Similar, any expression can be used with pinning. This results in a `PinnedExpressionNode`.
+
+              foo => ^(a + 1)
+
+          Anything else will result in the regular node for that expression, for example a `ConstantReadNode`.
+
+              foo => CONST
       - name: operator_loc
         type: location
+        comment: |
+          The location of the operator.
+
+              foo => bar
+                  ^^
     comment: |
       Represents the use of the `=>` operator.
 
@@ -3877,12 +4075,32 @@ nodes:
       - name: expression
         type: node
         kind: non-void expression
+        comment: |
+          The expression used in the pinned expression
+
+              foo in ^(bar)
+                       ^^^
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `^` operator
+
+              foo in ^(bar)
+                     ^
       - name: lparen_loc
         type: location
+        comment: |
+          The location of the opening parenthesis.
+
+              foo in ^(bar)
+                      ^
       - name: rparen_loc
         type: location
+        comment: |
+          The location of the closing parenthesis.
+
+              foo in ^(bar)
+                          ^
     comment: |
       Represents the use of the `^` operator for pinning an expression in a pattern matching expression.
 
@@ -3901,8 +4119,18 @@ nodes:
           - NumberedReferenceReadNode # foo in ^$1
           - ItLocalVariableReadNode # proc { 1 in ^it }
           - on error: MissingNode # foo in ^Bar
+        comment: |
+          The variable used in the pinned expression
+
+              foo in ^bar
+                      ^^^
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `^` operator
+
+              foo in ^bar
+                     ^
     comment: |
       Represents the use of the `^` operator for pinning a variable in a pattern matching expression.
 

data/docs/design.md

@@ -18,11 +18,11 @@ The templated files contain all of the code required to allocate and initialize
 
 In order to provide the best possible error tolerance, the parser is hand-written. It is structured using Pratt parsing, a technique developed by Vaughan Pratt back in the 1970s. Below are a bunch of links to articles and papers that explain Pratt parsing in more detail.
 
-* https://web.archive.org/web/20151223215421/http://hall.org.ua/halls/wizzard/pdf/Vaughan.Pratt.TDOP.pdf
+* https://github.com/tdop/tdop.github.io/raw/master/original.pdf
 * https://tdop.github.io/
 * https://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/
 * https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html
-* https://chidiwilliams.com/post/on-recursive-descent-and-pratt-parsing/
+* https://chidiwilliams.com/posts/on-recursive-descent-and-pratt-parsing
 
 You can find most of the functions that correspond to constructs in the Pratt parsing algorithm in `prism.c`. As a couple of examples: