yarp 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81d6e0266ee0d13563faafa47a0c0d09c5feabe870e2324f32564a7f68299dff
-  data.tar.gz: 771e9fd99caaf99cf7493d897cf9fdd5f93824e90bf09c22a1e5b5ec0e1a4bcf
+  metadata.gz: ef0d177f2be8f0d5ecd0ba54901e3cdf17fd6ec6f44a1b04160082ae5a1cb519
+  data.tar.gz: 776a4c7b981d4d6f3848bef030a448ea7d402f6ea75c20fcd87d3cbf935ecfe5
 SHA512:
-  metadata.gz: 16525d9ed87d00d9fd94af5cab5f99b8278ac4a81c7117195e80d3cd3cc6b12358fd0f2255cbe7df15ef1adef609719371bc3a210d8a2273d6e34fbdcb4b1a58
-  data.tar.gz: efbf02ccd9a7cb6a13b6ba56dc8fc3fc6e425d7d779583614fc9c24c74216fe3d89eadc83999f197a7227bfd90ca8ba9446c1b71c128e4874609f44f0869eb72
+  metadata.gz: 2e462ba8f51d24c8164ca67804a372d1bea6f6706828801cb2404af02ee183c301e221fef21725d59e013a64652eab06eae84b3e10a427f58ee7e0eed35c3950
+  data.tar.gz: 9594d53c55fc434c92c0601db17c548f473ca62812a0ab57b229d5b9753315198742c83f57a44744b6572def60b4b9659cfa1f93e4d86d66fd6b2f73e0db5234

data/CHANGELOG.md

@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [0.8.0] - 2023-08-18
+
+### Added
+
+- Some performance improvements when converting from the C AST to the Ruby AST.
+- Two rust crates have been added: `yarp-sys` and `yarp`. They are as yet unpublished.
+
+### Changed
+
+- Escaped newlines in strings and heredocs are now handled more correctly.
+- Dedenting heredocs that result in empty string nodes will now drop those string nodes from the list.
+- Beginless and endless ranges in conditional expressions now properly form a flip flop node.
+- `%` at the end of files no longer crashes.
+- Location information has been corrected for `if/elsif` chains that have no `else`.
+- `__END__` at the very end of the file was previously parsed as an identifier, but is now correct.
+- **BREAKING**: Nodes that reference `&&=`, `||=`, and other writing operators have been consolidated. Previously, they were separate individual nodes. Now they are a tree with the target being the left-hand side and the value being the right-hand side with a joining `AndWriteNode`, `OrWriteNode`, or `OperatorWriteNode` in the middle. This impacts all of the nodes that match this pattern: `{ClassVariable,Constant,ConstantPath,GlobalVariable,InstanceVariable,LocalVariable}Operator{And,Or,}WriteNode`.
+- **BREAKING**: `BlockParametersNode`, `ClassNode`, `DefNode`, `LambdaNode`, `ModuleNode`, `ParenthesesNode`, and `SingletonClassNode` have had their `statements` field renamed to `body` to give a hint that it might not be a `StatementsNode` (it could also be a `BeginNode`).
+
 ## [0.7.0] - 2023-08-14
 
 ### Added
@@ -22,7 +40,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - Autotools has been removed from the build system, so when the gem is installed it will no longer need to go through a configure step.
 - The AST for `foo = *bar` has changed to have an explicit array on the right hand side, rather than a splat node. This is more consistent with how other parsers handle this.
-- `RangeNodeFlags` has been renamed to `RangeFlags`.
+- **BREAKING**: `RangeNodeFlags` has been renamed to `RangeFlags`.
 - Unary minus on number literals is now parsed as part of the literal, rather than a call to a unary operator. This is more consistent with how other parsers handle this.
 
 ## [0.6.0] - 2023-08-09
@@ -31,6 +49,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/yarp/compare/v0.7.0...HEAD
+[unreleased]: https://github.com/ruby/yarp/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/ruby/yarp/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/ruby/yarp/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/ruby/yarp/compare/d60531...v0.6.0

data/Makefile

@@ -61,6 +61,8 @@ fuzz-docker-build: fuzz/docker/Dockerfile
 	$(Q) docker build -t yarp/fuzz fuzz/docker/
 
 fuzz-run-%: FORCE fuzz-docker-build
+	$(ECHO) "generating templates"
+	$(Q) bundle exec rake templates
 	$(ECHO) "running $* fuzzer"
 	$(Q) docker run --rm -v $(shell pwd):/yarp yarp/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
 	$(ECHO) "starting AFL++ run"

data/README.md

@@ -29,6 +29,8 @@ The repository contains the infrastructure for both a shared library (librubypar
 │   ├── yarp              Ruby library files
 │   └── yarp.rb           main entrypoint for the Ruby library
 ├── rakelib               various Rake tasks for the project
+├── rust
+│   └── yarp-sys          FFI binding for Rust
 ├── src
 │   ├── enc               various encoding files
 │   ├── util              various utility files
@@ -45,7 +47,7 @@ The repository contains the infrastructure for both a shared library (librubypar
 To compile the shared library, you will need:
 
 * A C99 compiler
-* autotools (autoconf, automake, libtool)
+* autotools autoconf, automake, libtool)
 * make
 * Ruby 3.3.0-preview1 or later
 
@@ -77,10 +79,11 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio
 * [Configuration](docs/configuration.md)
 * [Design](docs/design.md)
 * [Encoding](docs/encoding.md)
-* [Extension](docs/extension.md)
 * [Fuzzing](docs/fuzzing.md)
 * [Heredocs](docs/heredocs.md)
 * [Mapping](docs/mapping.md)
 * [Ripper](docs/ripper.md)
+* [Ruby API](docs/ruby_api.md)
 * [Serialization](docs/serialization.md)
 * [Testing](docs/testing.md)
+

data/config.yml

@@ -399,6 +399,19 @@ nodes:
 
           left and right
           ^^^^^^^^^^^^^^
+  - name: AndWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: value
+        type: node
+      - name: operator_loc
+        type: location
+    comment: |
+      Represents the use of the `&&=` operator.
+
+          target &&= value
+          ^^^^^^^^^^^^^^^^
   - name: ArgumentsNode
     child_nodes:
       - name: arguments
@@ -527,7 +540,7 @@ nodes:
       - name: parameters
         type: node?
         kind: BlockParametersNode
-      - name: statements
+      - name: body
         type: node?
       - name: opening_loc
         type: location
@@ -714,7 +727,7 @@ nodes:
         type: location?
       - name: superclass
         type: node?
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -723,47 +736,6 @@ nodes:
 
           class Foo end
           ^^^^^^^^^^^^^
-  - name: ClassVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a class variable.
-
-          @@target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: ClassVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a class variable.
-
-          @@target ||= value
-          ^^^^^^^^^^^^^^^^^^
-  - name: ClassVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a class variable using an operator that isn't `=`.
-
-          @@target += value
-          ^^^^^^^^^^^^^^^^^
   - name: ClassVariableReadNode
     comment: |
       Represents referencing a class variable.
@@ -783,47 +755,6 @@ nodes:
 
           @@foo = 1
           ^^^^^^^^^
-  - name: ConstantOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a constant.
-
-          Target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: ConstantOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a constant.
-
-          Target ||= value
-          ^^^^^^^^^^^^^^^^
-  - name: ConstantOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a constant using an operator that isn't `=`.
-
-          Target += value
-          ^^^^^^^^^^^^^^^
   - name: ConstantPathNode
     child_nodes:
       - name: parent
@@ -837,50 +768,6 @@ nodes:
 
           Foo::Bar
           ^^^^^^^^
-  - name: ConstantPathOperatorAndWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a constant path.
-
-          Parent::Child &&= value
-          ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: ConstantPathOperatorOrWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a constant path.
-
-          Parent::Child ||= value
-          ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: ConstantPathOperatorWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a constant path using an operator that isn't `=`.
-
-          Parent::Child += value
-          ^^^^^^^^^^^^^^^^^^^^^^
   - name: ConstantPathWriteNode
     child_nodes:
       - name: target
@@ -929,7 +816,7 @@ nodes:
       - name: parameters
         type: node?
         kind: ParametersNode
-      - name: statements
+      - name: body
         type: node?
       - name: locals
         type: constant[]
@@ -1123,47 +1010,6 @@ nodes:
 
           super
           ^^^^^
-  - name: GlobalVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a global variable.
-
-          $target &&= value
-          ^^^^^^^^^^^^^^^^^
-  - name: GlobalVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a global variable.
-
-          $target ||= value
-          ^^^^^^^^^^^^^^^^^
-  - name: GlobalVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a global variable using an operator that isn't `=`.
-
-          $target += value
-          ^^^^^^^^^^^^^^^^
   - name: GlobalVariableReadNode
     comment: |
       Represents referencing a global variable.
@@ -1263,47 +1109,6 @@ nodes:
 
           case a; in b then c end
                   ^^^^^^^^^^^
-  - name: InstanceVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to an instance variable.
-
-          @target &&= value
-          ^^^^^^^^^^^^^^^^^
-  - name: InstanceVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to an instance variable.
-
-          @target ||= value
-          ^^^^^^^^^^^^^^^^^
-  - name: InstanceVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to an instance variable using an operator that isn't `=`.
-
-          @target += value
-          ^^^^^^^^^^^^^^^^
   - name: InstanceVariableReadNode
     comment: |
       Represents referencing an instance variable.
@@ -1434,60 +1239,13 @@ nodes:
       - name: parameters
         type: node?
         kind: BlockParametersNode
-      - name: statements
+      - name: body
         type: node?
     comment: |
       Represents using a lambda literal (not the lambda method call).
 
           ->(value) { value * 2 }
           ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a local variable.
-
-          target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-    comment: |
-      Represents the use of the `||=` operator for assignment to a local variable.
-
-          target ||= value
-          ^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-      - name: operator_id
-        type: constant
-    comment: |
-      Represents assigning to a local variable using an operator that isn't `=`.
-
-          target += value
-          ^^^^^^^^^^^^^^^
   - name: LocalVariableReadNode
     child_nodes:
       - name: constant_id
@@ -1556,7 +1314,7 @@ nodes:
         type: location
       - name: constant_path
         type: node
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -1618,6 +1376,21 @@ nodes:
 
           $1
           ^^
+  - name: OperatorWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: operator_loc
+        type: location
+      - name: operator
+        type: constant
+      - name: value
+        type: node
+    comment: |
+      Represents the use of an operator on a write.
+
+          target += value
+          ^^^^^^^^^^^^^^^
   - name: OptionalParameterNode
     child_nodes:
       - name: constant_id
@@ -1647,6 +1420,19 @@ nodes:
 
           left or right
           ^^^^^^^^^^^^^
+  - name: OrWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: value
+        type: node
+      - name: operator_loc
+        type: location
+    comment: |
+      Represents the use of the `||=` operator.
+
+          target ||= value
+          ^^^^^^^^^^^^^^^^
   - name: ParametersNode
     child_nodes:
       - name: requireds
@@ -1673,7 +1459,7 @@ nodes:
           end
   - name: ParenthesesNode
     child_nodes:
-      - name: statements
+      - name: body
         type: node?
       - name: opening_loc
         type: location
@@ -1916,7 +1702,7 @@ nodes:
         type: location
       - name: expression
         type: node
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -1932,7 +1718,6 @@ nodes:
           __ENCODING__
           ^^^^^^^^^^^^
   - name: SourceFileNode
-    is_migrated: true
     child_nodes:
       - name: filepath
         type: string
@@ -2111,10 +1896,12 @@ nodes:
         type: node?
         kind: StatementsNode
     comment: |
-      case true
-      when true
-      ^^^^^^^^^
-      end
+      Represents the use of the `when` keyword within a case statement.
+
+          case true
+          when true
+          ^^^^^^^^^
+          end
   - name: WhileNode
     child_nodes:
       - name: keyword_loc