prism 0.29.0 → 0.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ac8167684c0f975fdba449e77206d97756f6c39f22ce871f79a79a61260503f
-  data.tar.gz: 6a2dd5c0a47df7c8c575ad2e5b344c789d548e93cfe82e5fa29974d46a52bb3c
+  metadata.gz: 46cabe6b76e675b905cb9b2941faf00d35c0c5043e480c094952a89404e87587
+  data.tar.gz: 50a764fb701657e5936b764a7438f49ab074f45d5bee3e435bf70c9b891067be
 SHA512:
-  metadata.gz: dfa7fe63285b85cb45aa0be681916d6891ee5e5adf2162ab5c18423417f7938afc99f00eca8fd6187fbd0b7168af088d932fbda3c1361d1c73953f39329bed70
-  data.tar.gz: 79bc51db60600d74a6bfbeae5eba3a8ec505e9809d0f7ff0b05563f1f7a8cd284cc293233397323a443217a4a5271f90b4c40516168ae3a8be0ddc2d7376cef9
+  metadata.gz: 58535a5049cf8d6b8ceb97721135f58ca2fa4e7311cce8c336b6d4d5674de38861a07728651859ffe926bf84db8c14454b543d5d26732174de4c67a2be7cb46b
+  data.tar.gz: 16ae75556351bd4c96a54dbd930e466c40695419c29ba9cf5b048dc1173fc397094974bea313258e7615a27f67c9b55b00d72dcee33ddacc1613169005b0e184

data/CHANGELOG.md

@@ -6,6 +6,26 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [0.30.0] - 2024-06-07
+
+### Added
+
+- More correctly raise mixed encoding errors.
+- Implement ambiguous binary operator warning.
+- Fix up regexp escapes with control and meta characters.
+- Fix up support for the `it` implicit local variable.
+- Heredoc identifiers now properly disallow CLRF.
+- Errors added for void value expressions in begin clauses.
+- Many updates to more closely match the `parser` gem in parser translation.
+- Many errors added for invalid regular expressions.
+
+### Changed
+
+- Handle parser translation missing the `parser` gem.
+- Handle ruby_parser translation missing the `ruby_parser` gem.
+- Various error messages have been updated to more closely match CRuby.
+- `RationalNode` now has a `numerator` and `denominator` field instead of a `numeric` field. For the Ruby API we provide a `RationalNode#numeric` method for backwards-compatibility.
+
 ## [0.29.0] - 2024-05-10
 
 ### Added
@@ -518,7 +538,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v0.29.0...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v0.30.0...HEAD
+[0.30.0]: https://github.com/ruby/prism/compare/v0.29.0...v0.30.0
 [0.29.0]: https://github.com/ruby/prism/compare/v0.28.0...v0.29.0
 [0.28.0]: https://github.com/ruby/prism/compare/v0.27.0...v0.28.0
 [0.27.0]: https://github.com/ruby/prism/compare/v0.26.0...v0.27.0

data/CONTRIBUTING.md

@@ -10,10 +10,6 @@ The discussions page on the GitHub repository are open. If you have a question o
 
 If you want to contribute code, please first open or contribute to a discussion. A lot of the project is in flux, and we want to make sure that you are contributing to the right place. Once you have a discussion going, you can open a pull request with your changes. We will review your code and get it merged in.
 
-### Ruby Features
-
-Pattern matching and endless method definitions should be avoided as long as the latest TruffleRuby release does not support it.
-
 ## Tests
 
 We could always use more tests! One of the biggest challenges of this project is building up a big test suite. If you want to contribute tests, feel free to open a pull request. These will get merged in as soon as possible.

data/README.md

@@ -40,6 +40,7 @@ The repository contains the infrastructure for both a shared library (libprism)
 ├── rust
 │   ├── ruby-prism        Rustified crate for the shared library
 │   └── ruby-prism-sys    FFI binding for Rust
+├── sample                Sample code that uses the Ruby API for documentation
 ├── sig                   RBS type signatures for the Ruby library
 ├── src
 │   ├── util              various utility files

data/config.yml

@@ -114,6 +114,7 @@ errors:
   - EXPRESSION_NOT_WRITABLE_FILE
   - EXPRESSION_NOT_WRITABLE_LINE
   - EXPRESSION_NOT_WRITABLE_NIL
+  - EXPRESSION_NOT_WRITABLE_NUMBERED
   - EXPRESSION_NOT_WRITABLE_SELF
   - EXPRESSION_NOT_WRITABLE_TRUE
   - FLOAT_PARSE
@@ -153,6 +154,7 @@ errors:
   - INVALID_NUMBER_UNDERSCORE_INNER
   - INVALID_NUMBER_UNDERSCORE_TRAILING
   - INVALID_PERCENT
+  - INVALID_PERCENT_EOF
   - INVALID_PRINTABLE_CHARACTER
   - INVALID_RETRY_AFTER_ELSE
   - INVALID_RETRY_AFTER_ENSURE
@@ -184,9 +186,10 @@ errors:
   - NO_LOCAL_VARIABLE
   - NOT_EXPRESSION
   - NUMBER_LITERAL_UNDERSCORE
+  - NUMBERED_PARAMETER_INNER_BLOCK
   - NUMBERED_PARAMETER_IT
   - NUMBERED_PARAMETER_ORDINARY
-  - NUMBERED_PARAMETER_OUTER_SCOPE
+  - NUMBERED_PARAMETER_OUTER_BLOCK
   - OPERATOR_MULTI_ASSIGN
   - OPERATOR_WRITE_ARGUMENTS
   - OPERATOR_WRITE_BLOCK
@@ -203,8 +206,8 @@ errors:
   - PARAMETER_SPLAT_MULTI
   - PARAMETER_STAR
   - PARAMETER_UNEXPECTED_FWD
-  - PARAMETER_WILD_LOOSE_COMMA
   - PARAMETER_UNEXPECTED_NO_KW
+  - PARAMETER_WILD_LOOSE_COMMA
   - PATTERN_CAPTURE_DUPLICATE
   - PATTERN_EXPRESSION_AFTER_BRACKET
   - PATTERN_EXPRESSION_AFTER_COMMA
@@ -233,6 +236,7 @@ errors:
   - REGEXP_INCOMPAT_CHAR_ENCODING
   - REGEXP_INVALID_UNICODE_RANGE
   - REGEXP_NON_ESCAPED_MBC
+  - REGEXP_PARSE_ERROR
   - REGEXP_TERM
   - REGEXP_UNKNOWN_OPTIONS
   - REGEXP_UTF8_CHAR_NON_UTF8_REGEXP
@@ -273,6 +277,7 @@ errors:
   - WRITE_TARGET_UNEXPECTED
   - XSTRING_TERM
 warnings:
+  - AMBIGUOUS_BINARY_OPERATOR
   - AMBIGUOUS_FIRST_ARGUMENT_MINUS
   - AMBIGUOUS_FIRST_ARGUMENT_PLUS
   - AMBIGUOUS_PREFIX_AMPERSAND
@@ -1876,19 +1881,56 @@ nodes:
     fields:
       - name: index
         type: node
+        comment: |
+          The index expression for `for` loops.
+
+              for i in a end
+                  ^
       - name: collection
         type: node
+        comment: |
+          The collection to iterate over.
+
+              for i in a end
+                       ^
       - name: statements
         type: node?
         kind: StatementsNode
+        comment: |
+          Represents the body of statements to execute for each iteration of the loop.
+
+              for i in a
+                foo(i)
+                ^^^^^^
+              end
       - name: for_keyword_loc
         type: location
+        comment: |
+          The location of the `for` keyword.
+
+              for i in a end
+              ^^^
       - name: in_keyword_loc
         type: location
+        comment: |
+          The location of the `in` keyword.
+
+              for i in a end
+                    ^^
       - name: do_keyword_loc
         type: location?
+        comment: |
+          The location of the `do` keyword, if present.
+
+              for i in a do end
+                         ^^
       - name: end_keyword_loc
         type: location
+        comment: |
+          The location of the `end` keyword.
+
+              for i in a end
+                         ^^^
     comment: |
       Represents the use of the `for` keyword.
 
@@ -2566,6 +2608,12 @@ nodes:
 
           `foo #{bar} baz`
           ^^^^^^^^^^^^^^^^
+  - name: ItLocalVariableReadNode
+    comment: |
+      Represents reading from the implicit `it` local variable.
+
+          -> { it }
+               ^^
   - name: ItParametersNode
     comment: |
       Represents an implicit set of parameters through the use of the `it` keyword within a block or lambda.
@@ -2691,10 +2739,6 @@ nodes:
 
               _1     # name `:_1`
 
-          Finally, for the default `it` block parameter, the name is `0it`. This is to distinguish it from an `it` local variable that is explicitly declared.
-
-              it     # name `:0it`
-
       - name: depth
         type: uint32
         comment: |
@@ -3227,8 +3271,21 @@ nodes:
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   - name: RationalNode
     fields:
-      - name: numeric
-        type: node
+      - name: flags
+        type: flags
+        kind: IntegerBaseFlags
+      - name: numerator
+        type: integer
+        comment: |
+          The numerator of the rational number.
+
+              1.5r # numerator 3
+      - name: denominator
+        type: integer
+        comment: |
+          The denominator of the rational number.
+
+              1.5r # denominator 2
     comment: |
       Represents a rational number literal.
 
@@ -3567,7 +3624,7 @@ nodes:
                          ^^^^
       - name: then_keyword_loc
         type: location?
-        comment:
+        comment: |
           The location of the `then` keyword, if present.
 
               unless cond then bar end

data/docs/fuzzing.md

@@ -26,7 +26,7 @@ fuzz
 There are currently three fuzzing targets
 
 - `pm_serialize_parse` (parse)
-- `pm_regexp_named_capture_group_names` (regexp)
+- `pm_regexp_parse` (regexp)
 
 Respectively, fuzzing can be performed with
 

data/docs/ripper_translation.md

@@ -48,3 +48,25 @@ ArithmeticRipper.new("1 + 2 - 3").parse # => [0]
 ```
 
 The exact names of the `on_*` methods are listed in the `Ripper` source.
+
+## Background
+
+It is helpful to understand the differences between the `Ripper` library and the `Prism` library. Both libraries perform parsing and provide you with APIs to manipulate and understand the resulting syntax tree. However, there are a few key differences.
+
+### Design
+
+`Ripper` is a streaming parser. This means as it is parsing Ruby code, it dispatches events back to the consumer. This allows quite a bit of flexibility. You can use it to build your own syntax tree or to find specific patterns in the code. `Prism` on the other hand returns to your the completed syntax tree _before_ it allows you to manipulate it. This means the tree that you get back is the only representation that can be generated by the parser _at parse time_ (but of course can be manipulated later).
+
+### Fields
+
+We use the term "field" to mean a piece of information on a syntax tree node. `Ripper` provides the minimal number of fields to accurately represent the syntax tree for the purposes of compilation/interpretation. For example, in the callbacks for nodes that are based on keywords (`class`, `module`, `for`, `while`, etc.) you are not given the keyword itself, you need to attach it on your own. In other cases, tokens are not necessarily dispatched at all, meaning you need to find them yourself. `Prism` provides the opposite: the maximum number of fields on nodes is provided. As a tradeoff, this requires more memory, but this is chosen to make it easier on consumers.
+
+### Maintainability
+
+The `Ripper` interface is not guaranteed in any way, and tends to change between patch versions of CRuby. This is largely due to the fact that `Ripper` is a by-product of the generated parser, as opposed to its own parser. As an example, in the expression `foo::bar = baz`, there are three different represents possible for the call operator, including:
+
+* `:"::"` - Ruby 1.9 to Ruby 3.1.4
+* `73` - Ruby 3.1.5 to Ruby 3.1.6
+* `[:@op, "::", [lineno, column]]` - Ruby 3.2.0 and later
+
+The `Prism` interface is guaranteed going forward to be the consistent, and the official Ruby syntax tree interface. This means you can rely on this interface without having to worry about individual changes between Ruby versions. It also is a gem, which means it is versioned based on the gem version, as opposed to being versioned based on the Ruby version. Finally, you can use `Prism` to parse multiple versions of Ruby, whereas `Ripper` is tied to the Ruby version it is running on.

data/ext/prism/api_node.c

@@ -102,6 +102,7 @@ static VALUE rb_cPrismInterpolatedRegularExpressionNode;
 static VALUE rb_cPrismInterpolatedStringNode;
 static VALUE rb_cPrismInterpolatedSymbolNode;
 static VALUE rb_cPrismInterpolatedXStringNode;
+static VALUE rb_cPrismItLocalVariableReadNode;
 static VALUE rb_cPrismItParametersNode;
 static VALUE rb_cPrismKeywordHashNode;
 static VALUE rb_cPrismKeywordRestParameterNode;
@@ -980,12 +981,6 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     pm_node_stack_push(&node_stack, (pm_node_t *) cast->right);
                     break;
                 }
-#line 146 "api_node.c.erb"
-                case PM_RATIONAL_NODE: {
-                    pm_rational_node_t *cast = (pm_rational_node_t *) node;
-                    pm_node_stack_push(&node_stack, (pm_node_t *) cast->numeric);
-                    break;
-                }
 #line 146 "api_node.c.erb"
                 case PM_RESCUE_MODIFIER_NODE: {
                     pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
@@ -3733,6 +3728,19 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismInterpolatedXStringNode));
                     break;
                 }
+#line 172 "api_node.c.erb"
+                case PM_IT_LOCAL_VARIABLE_READ_NODE: {
+                    VALUE argv[2];
+
+                    // source
+                    argv[0] = source;
+
+                    // location
+                    argv[1] = pm_location_new(parser, node->location.start, node->location.end);
+
+                    rb_ary_push(value_stack, rb_class_new_instance(2, argv, rb_cPrismItLocalVariableReadNode));
+                    break;
+                }
 #line 172 "api_node.c.erb"
                 case PM_IT_PARAMETERS_NODE: {
                     VALUE argv[2];
@@ -4723,19 +4731,28 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                 }
 #line 172 "api_node.c.erb"
                 case PM_RATIONAL_NODE: {
-                    VALUE argv[3];
+                    pm_rational_node_t *cast = (pm_rational_node_t *) node;
+                    VALUE argv[5];
 
                     // source
                     argv[0] = source;
 
-                    // numeric
-#line 186 "api_node.c.erb"
-                    argv[1] = rb_ary_pop(value_stack);
+                    // flags
+#line 223 "api_node.c.erb"
+                    argv[1] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK);
+
+                    // numerator
+#line 226 "api_node.c.erb"
+                    argv[2] = pm_integer_new(&cast->numerator);
+
+                    // denominator
+#line 226 "api_node.c.erb"
+                    argv[3] = pm_integer_new(&cast->denominator);
 
                     // location
-                    argv[2] = pm_location_new(parser, node->location.start, node->location.end);
+                    argv[4] = pm_location_new(parser, node->location.start, node->location.end);
 
-                    rb_ary_push(value_stack, rb_class_new_instance(3, argv, rb_cPrismRationalNode));
+                    rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismRationalNode));
                     break;
                 }
 #line 172 "api_node.c.erb"
@@ -5578,6 +5595,7 @@ Init_prism_api_node(void) {
     rb_cPrismInterpolatedStringNode = rb_define_class_under(rb_cPrism, "InterpolatedStringNode", rb_cPrismNode);
     rb_cPrismInterpolatedSymbolNode = rb_define_class_under(rb_cPrism, "InterpolatedSymbolNode", rb_cPrismNode);
     rb_cPrismInterpolatedXStringNode = rb_define_class_under(rb_cPrism, "InterpolatedXStringNode", rb_cPrismNode);
+    rb_cPrismItLocalVariableReadNode = rb_define_class_under(rb_cPrism, "ItLocalVariableReadNode", rb_cPrismNode);
     rb_cPrismItParametersNode = rb_define_class_under(rb_cPrism, "ItParametersNode", rb_cPrismNode);
     rb_cPrismKeywordHashNode = rb_define_class_under(rb_cPrism, "KeywordHashNode", rb_cPrismNode);
     rb_cPrismKeywordRestParameterNode = rb_define_class_under(rb_cPrism, "KeywordRestParameterNode", rb_cPrismNode);