prism 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72e48ffbc7a58cb622a709f860470c4cf2178113a2ba6a2c9cd4f838229ccff8
-  data.tar.gz: 709ecc0c274a141a868661f94d28aad3fda3032e1b253b2f699a15ff71200c37
+  metadata.gz: 742f60637c4fd77f62b7713f484e87d70249e6e9dfeefc2c2ba0bfe667eed034
+  data.tar.gz: 67bd239271c1d848536389668722a0617419b19b6bb250ed39cf17b7a414da4c
 SHA512:
-  metadata.gz: 42a21574ff5ca1ce62c836839a8c54ed7d6a3f943cfbbd13d860aa30f3d1cf2b6fd5add3082d7622b30e3403ecbf98a404f3c9ef831744e60d40deabeae44ba2
-  data.tar.gz: 8ef39e9818ba2f13808a0e1131514709e6c7cbbbf2a0c784d047637d56b0024d35951a852c91d8f29c72477b5e923a6cc2687335cf44ccc761ab2e8e5eb5e521
+  metadata.gz: d0a90337f2635d35b08b0932ad6d928610406bb3f908c1b7b601f5fcb08b404604745f93bffd9a4bb84fc13cde0b6b4a71015390546a077daa4e05d7d8cf965e
+  data.tar.gz: 231693786022302c486d3c4ea2c8841636e3c94cb37f6b5e410f1ab6ac4ce7fc12e53cd4d67f43d6a38f3292867fda808e655391528619d626823a5163cbf722

data/CHANGELOG.md

@@ -6,6 +6,33 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [0.20.0] - 2024-01-01
+
+### Added
+
+- String literal hash keys are now marked as frozen as static literal.
+- `IndexTargetNode` now always has the `ATTRIBUTE_WRITE` flag.
+- `Call*Node` nodes now have an `IGNORE_VISIBILITY` flag.
+- We now support the `it` default parameter.
+- Errors and warnings now have levels associated with them.
+- Symbols now have correct encoding flags.
+- We have now merged `parser-prism` in, which provides translation to the `whitequark/parser` AST.
+- We now emit errors for invalid method definition receivers.
+
+### Changed
+
+- We now emit errors on invalid pinned local variables.
+- When passed scopes, it is now assumed that the innermost scope is the current binding.
+- We now provide better error recovery for non terminated heredocs.
+- Fix for `RationalNode#value` for non-decimal integers.
+- Unary symbols `!@` and `~@` now unescape to `!` and `~`, respectively.
+- `frozen_string_literal: false` now works properly.
+
+### Removed
+
+- We've removed the `locals_body_index` field.
+- We've removed the `verbose` option on the various parse APIs. Warnings are now always emitted with their associated level so that consumers can decide how to handle them.
+
 ## [0.19.0] - 2023-12-14
 
 ### Added
@@ -296,7 +323,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v0.19.0...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v0.20.0...HEAD
+[0.20.0]: https://github.com/ruby/prism/compare/v0.19.0...v0.20.0
 [0.19.0]: https://github.com/ruby/prism/compare/v0.18.0...v0.19.0
 [0.18.0]: https://github.com/ruby/prism/compare/v0.17.1...v0.18.0
 [0.17.1]: https://github.com/ruby/prism/compare/v0.17.0...v0.17.1

data/Makefile

@@ -25,6 +25,7 @@ all: shared static
 shared: build/libprism.$(SOEXT)
 static: build/libprism.a
 wasm: javascript/src/prism.wasm
+java-wasm: java-wasm/src/test/resources/prism.wasm
 
 build/libprism.$(SOEXT): $(SHARED_OBJECTS)
 	$(ECHO) "linking $@"
@@ -38,6 +39,10 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
 	$(Q) $(WASI_SDK_PATH)/bin/clang --sysroot=$(WASI_SDK_PATH)/share/wasi-sysroot/ $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -o $@ $(SOURCES)
 
+java-wasm/src/test/resources/prism.wasm: Makefile $(SOURCES) $(HEADERS)
+	$(ECHO) "building $@"
+	$(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES)
+
 build/shared/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"
 	$(Q) mkdir -p $(@D)

data/README.md

@@ -32,8 +32,8 @@ The repository contains the infrastructure for both a shared library (libprism)
 │   └── prism.rb          main entrypoint for the Ruby library
 ├── rakelib               various Rake tasks for the project
 ├── rust
-│   ├── prism             Rustified crate for the shared library
-│   └── prism-sys         FFI binding for Rust
+│   ├── ruby-prism        Rustified crate for the shared library
+│   └── ruby-prism-sys    FFI binding for Rust
 ├── src
 │   ├── enc               various encoding files
 │   ├── util              various utility files
@@ -50,10 +50,9 @@ The repository contains the infrastructure for both a shared library (libprism)
 
 To compile the shared library, you will need:
 
-* A C99 compiler
-* autotools (autoconf, automake, libtool)
+* C99 compiler
 * make
-* Ruby 3.3.0-preview1 or later
+* Ruby 3.0.0 or later
 
 Once you have these dependencies, run:
 
@@ -79,8 +78,9 @@ to see the syntax tree for the expression `1 + 2`.
 
 See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additionally have documentation about the overall design of the project as well as various subtopics.
 
-* [Building](docs/building.md)
+* [Build system](docs/build_system.md)
 * [Configuration](docs/configuration.md)
+* [CRuby compilation](docs/cruby_compilation.md)
 * [Design](docs/design.md)
 * [Encoding](docs/encoding.md)
 * [Fuzzing](docs/fuzzing.md)
@@ -88,6 +88,8 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio
 * [JavaScript](docs/javascript.md)
 * [Local variable depth](docs/local_variable_depth.md)
 * [Mapping](docs/mapping.md)
+* [Parser translation](docs/parser_translation.md)
+* [Parsing rules](docs/parsing_rules.md)
 * [Releasing](docs/releasing.md)
 * [Ripper](docs/ripper.md)
 * [Ruby API](docs/ruby_api.md)

data/config.yml

@@ -347,6 +347,8 @@ flags:
         comment: "a call that could have been a local variable"
       - name: ATTRIBUTE_WRITE
         comment: "a call that is an attribute write, so the value being written should be returned"
+      - name: IGNORE_VISIBILITY
+        comment: "a call that ignores method visibility"
     comment: Flags for call nodes.
   - name: EncodingFlags
     values:
@@ -368,14 +370,19 @@ flags:
     comment: Flags for integer nodes that correspond to the base of the integer.
   - name: KeywordHashNodeFlags
     values:
-      - name: STATIC_KEYS
-        comment: "a keyword hash which only has `AssocNode` elements all with static literal keys, which means the elements can be treated as keyword arguments"
+      - name: SYMBOL_KEYS
+        comment: "a keyword hash which only has `AssocNode` elements all with symbol keys, which means the elements can be treated as keyword arguments"
     comment: Flags for keyword hash nodes.
   - name: LoopFlags
     values:
       - name: BEGIN_MODIFIER
         comment: "a loop after a begin statement, so the body is executed first before the condition"
     comment: Flags for while and until loop nodes.
+  - name: ParameterFlags
+    values:
+      - name: REPEATED_PARAMETER
+        comment: "a parameter name that has been repeated in the method signature"
+    comment: Flags for parameter nodes.
   - name: RangeFlags
     values:
       - name: EXCLUDE_END
@@ -468,10 +475,31 @@ nodes:
     fields:
       - name: left
         type: node
+        comment: |
+          Represents the left side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              left and right
+              ^^^^
+
+              1 && 2
+              ^
       - name: right
         type: node
+        comment: |
+          Represents the right side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              left && right
+                      ^^^^^
+
+              1 and 2
+                    ^
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `and` keyword or the `&&` operator.
+
+              left and right
+                   ^^^
     comment: |
       Represents the use of the `&&` operator or the `and` keyword.
 
@@ -501,8 +529,7 @@ nodes:
       - name: closing_loc
         type: location?
     comment: |
-      Represents an array literal. This can be a regular array using brackets or
-      a special array using % like %w or %i.
+      Represents an array literal. This can be a regular array using brackets or a special array using % like %w or %i.
 
           [1, 2, 3]
           ^^^^^^^^^
@@ -541,10 +568,34 @@ nodes:
     fields:
       - name: key
         type: node
+        comment: |
+          The key of the association. This can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              { a: b }
+                ^
+
+              { foo => bar }
+                ^^^
+
+              { def a; end => 1 }
+                ^^^^^^^^^^
       - name: value
-        type: node?
+        type: node
+        comment: |
+          The value of the association, if present. This can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              { foo => bar }
+                       ^^^
+
+              { x: 1 }
+                   ^
       - name: operator_loc
         type: location?
+        comment: |
+          The location of the `=>` operator, if present.
+
+              { foo => bar }
+                    ^^
     comment: |
       Represents a hash key/value pair.
 
@@ -554,8 +605,18 @@ nodes:
     fields:
       - name: value
         type: node?
+        comment: |
+          The value to be splatted, if present. Will be missing when keyword rest argument forwarding is used.
+
+              { **foo }
+                  ^^^
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `**` operator.
+
+              { **x }
+                ^^
     comment: |
       Represents a splat in a hash literal.
 
@@ -565,6 +626,12 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the back-reference variable, including the leading `$`.
+
+              $& # name `:$&`
+
+              $+ # name `:$+`
     comment: |
       Represents reading a reference to a field in the previous match.
 
@@ -609,6 +676,9 @@ nodes:
           ^^^^^^^^^^
   - name: BlockLocalVariableNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant
     comment: |
@@ -620,8 +690,6 @@ nodes:
     fields:
       - name: locals
         type: constant[]
-      - name: locals_body_index
-        type: uint32
       - name: parameters
         type: node?
       - name: body
@@ -633,10 +701,13 @@ nodes:
     comment: |
       Represents a block of ruby code.
 
-      [1, 2, 3].each { |i| puts x }
-                     ^^^^^^^^^^^^^^
+          [1, 2, 3].each { |i| puts x }
+                         ^^^^^^^^^^^^^^
   - name: BlockParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant?
       - name: name_loc
@@ -712,6 +783,17 @@ nodes:
         kind: CallNodeFlags
       - name: receiver
         type: node?
+        comment: |
+          The object that the method is being called on. This can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              foo.bar
+              ^^^
+
+              +foo
+               ^^^
+
+              foo + bar
+              ^^^
       - name: call_operator_loc
         type: location?
       - name: name
@@ -950,6 +1032,12 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the class variable, which is a `@@` followed by an [identifier](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#identifiers).
+
+              @@abc   # name `:@@abc`
+
+              @@_test # name `:@@_test`
     comment: |
       Represents referencing a class variable.
 
@@ -1120,6 +1208,12 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the [constant](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#constants).
+
+              X              # name `:X`
+
+              SOME_CONSTANT  # name `:SOME_CONSTANT`
     comment: |
       Represents referencing a constant.
 
@@ -1164,8 +1258,6 @@ nodes:
         type: node?
       - name: locals
         type: constant[]
-      - name: locals_body_index
-        type: uint32
       - name: def_keyword_loc
         type: location
       - name: operator_loc
@@ -1407,6 +1499,12 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the global variable, which is a `$` followed by an [identifier](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#identifier). Alternatively, it can be one of the special global variables designated by a symbol.
+
+              $foo   # name `:$foo`
+
+              $_Test # name `:$_Test`
     comment: |
       Represents referencing a global variable.
 
@@ -1440,10 +1538,28 @@ nodes:
     fields:
       - name: opening_loc
         type: location
+        comment: |
+          The location of the opening brace.
+
+              { a => b }
+              ^
       - name: elements
         type: node[]
+        comment: |
+          The elements of the hash. These can be either `AssocNode`s or `AssocSplatNode`s.
+
+              { a: b }
+                ^^^^
+
+              { **foo }
+                ^^^^^
       - name: closing_loc
         type: location
+        comment: |
+          The location of the closing brace.
+
+              { a => b }
+                       ^
     comment: |
       Represents a hash literal.
 
@@ -1507,14 +1623,16 @@ nodes:
       - name: value
         type: node
     comment: |
-      Represents a node that is implicitly being added to the tree but doesn't
-      correspond directly to a node in the source.
+      Represents a node that is implicitly being added to the tree but doesn't correspond directly to a node in the source.
 
           { foo: }
             ^^^^
 
           { Foo: }
             ^^^^
+
+          foo in { bar: }
+                   ^^^^
   - name: ImplicitRestNode
     comment: |
       Represents using a trailing comma to indicate an implicit rest parameter.
@@ -1709,6 +1827,12 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the instance variable, which is a `@` followed by an [identifier](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#identifiers).
+
+              @x     # name `:@x`
+
+              @_test # name `:@_test`
     comment: |
       Represents referencing an instance variable.
 
@@ -1761,9 +1885,7 @@ nodes:
         type: location
     newline: parts
     comment: |
-      Represents a regular expression literal that contains interpolation that
-      is being used in the predicate of a conditional to implicitly match
-      against the last line read by an IO object.
+      Represents a regular expression literal that contains interpolation that is being used in the predicate of a conditional to implicitly match against the last line read by an IO object.
 
           if /foo #{bar} baz/ then end
              ^^^^^^^^^^^^^^^^
@@ -1840,6 +1962,9 @@ nodes:
               ^^^^
   - name: KeywordRestParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant?
       - name: name_loc
@@ -1856,8 +1981,6 @@ nodes:
     fields:
       - name: locals
         type: constant[]
-      - name: locals_body_index
-        type: uint32
       - name: operator_loc
         type: location
       - name: opening_loc
@@ -1930,12 +2053,33 @@ nodes:
     fields:
       - name: name
         type: constant
+        comment: |
+          The name of the local variable, which is an [identifier](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#identifiers).
+
+              x      # name `:x`
+
+              _Test  # name `:_Test`
+
+          Note that this can also be an underscore followed by a number for the default block parameters.
+
+              _1     # name `:_1`
+
+          Finally, for the default `it` block parameter, the name is `0it`. This is to distinguish it from an `it` local variable that is explicitly declared.
+
+              it     # name `:0it`
+
       - name: depth
         type: uint32
+        comment: |
+          The number of visible scopes that should be searched to find the origin of this local variable.
+
+              foo = 1; foo # depth 0
+
+              bar = 2; tap { bar } # depth 1
+
+          The specific rules for calculating the depth may differ from individual Ruby implementations, as they are not specified by the language. For more information, see [the Prism documentation](https://github.com/ruby/prism/blob/main/docs/local_variable_depth.md).
     comment: |
-      Represents reading a local variable. Note that this requires that a local
-      variable of the same name has already been written to in the same scope,
-      otherwise it is parsed as a method call.
+      Represents reading a local variable. Note that this requires that a local variable of the same name has already been written to in the same scope, otherwise it is parsed as a method call.
 
           foo
           ^^^
@@ -1981,9 +2125,7 @@ nodes:
       - name: unescaped
         type: string
     comment: |
-      Represents a regular expression literal used in the predicate of a
-      conditional to implicitly match against the last line read by an IO
-      object.
+      Represents a regular expression literal used in the predicate of a conditional to implicitly match against the last line read by an IO object.
 
           if /foo/i then end
              ^^^^^^
@@ -2021,15 +2163,13 @@ nodes:
       - name: targets
         type: node[]
     comment: |
-      Represents writing local variables using a regular expression match with
-      named capture groups.
+      Represents writing local variables using a regular expression match with named capture groups.
 
           /(?<foo>bar)/ =~ baz
           ^^^^^^^^^^^^^^^^^^^^
   - name: MissingNode
     comment: |
-      Represents a node that is missing from the source and results in a syntax
-      error.
+      Represents a node that is missing from the source and results in a syntax error.
   - name: ModuleNode
     fields:
       - name: locals
@@ -2122,8 +2262,7 @@ nodes:
       - name: maximum
         type: uint8
     comment: |
-      Represents an implicit set of parameters through the use of numbered
-      parameters within a block or lambda.
+      Represents an implicit set of parameters through the use of numbered parameters within a block or lambda.
 
           -> { _1 + _2 }
           ^^^^^^^^^^^^^^
@@ -2131,6 +2270,14 @@ nodes:
     fields:
       - name: number
         type: uint32
+        comment: |
+          The (1-indexed, from the left) number of the capture group. Numbered references that would overflow a `uint32`  result in a `number` of exactly `2**32 - 1`.
+
+              $1          # number `1`
+
+              $5432       # number `5432`
+
+              $4294967296 # number `4294967295`
     comment: |
       Represents reading a numbered reference to a capture in the previous match.
 
@@ -2138,6 +2285,9 @@ nodes:
           ^^
   - name: OptionalKeywordParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant
       - name: name_loc
@@ -2152,6 +2302,9 @@ nodes:
           end
   - name: OptionalParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant
       - name: name_loc
@@ -2170,10 +2323,31 @@ nodes:
     fields:
       - name: left
         type: node
+        comment: |
+          Represents the left side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              left or right
+              ^^^^
+
+              1 || 2
+              ^
       - name: right
         type: node
+        comment: |
+          Represents the right side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              left || right
+                      ^^^^^
+
+              1 or 2
+                   ^
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `or` keyword or the `||` operator.
+
+              left or right
+                   ^^
     comment: |
       Represents the use of the `||` operator or the `or` keyword.
 
@@ -2227,8 +2401,7 @@ nodes:
       - name: rparen_loc
         type: location
     comment: |
-      Represents the use of the `^` operator for pinning an expression in a
-      pattern matching expression.
+      Represents the use of the `^` operator for pinning an expression in a pattern matching expression.
 
           foo in ^(bar)
                  ^^^^^^
@@ -2239,8 +2412,7 @@ nodes:
       - name: operator_loc
         type: location
     comment: |
-      Represents the use of the `^` operator for pinning a variable in a pattern
-      matching expression.
+      Represents the use of the `^` operator for pinning a variable in a pattern matching expression.
 
           foo in ^bar
                  ^^^^
@@ -2291,10 +2463,29 @@ nodes:
         kind: RangeFlags
       - name: left
         type: node?
+        comment: |
+          The left-hand side of the range, if present. It can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              1...
+              ^
+
+              hello...goodbye
+              ^^^^^
       - name: right
         type: node?
+        comment: |
+          The right-hand side of the range, if present. It can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
+
+              ..5
+                ^
+
+              1...foo
+                  ^^^
+          If neither right-hand or left-hand side was included, this will be a MissingNode.
       - name: operator_loc
         type: location
+        comment: |
+          The location of the `..` or `...` operator.
     comment: |
       Represents the use of the `..` or `...` operators.
 
@@ -2338,6 +2529,9 @@ nodes:
           ^^^^^^
   - name: RequiredKeywordParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant
       - name: name_loc
@@ -2350,6 +2544,9 @@ nodes:
           end
   - name: RequiredParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant
     comment: |
@@ -2397,10 +2594,12 @@ nodes:
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           end
 
-      `Foo, *splat, Bar` are in the `exceptions` field.
-      `ex` is in the `exception` field.
+      `Foo, *splat, Bar` are in the `exceptions` field. `ex` is in the `exception` field.
   - name: RestParameterNode
     fields:
+      - name: flags
+        type: flags
+        kind: ParameterFlags
       - name: name
         type: constant?
       - name: name_loc
@@ -2511,8 +2710,7 @@ nodes:
       - name: unescaped
         type: string
     comment: |
-      Represents a string literal, a string contained within a `%w` list, or
-      plain string content within an interpolated string.
+      Represents a string literal, a string contained within a `%w` list, or plain string content within an interpolated string.
 
           "foo"
           ^^^^^