yarp 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51ee2a88319b74a9937ab37fbaa9bb71586caf26a8f31282c2d2431735a484b6
-  data.tar.gz: efe7ac9662cd199e3ba0c8a00f913912a2830843bc0c87cdfc3b258d724eafd8
+  metadata.gz: ef0d177f2be8f0d5ecd0ba54901e3cdf17fd6ec6f44a1b04160082ae5a1cb519
+  data.tar.gz: 776a4c7b981d4d6f3848bef030a448ea7d402f6ea75c20fcd87d3cbf935ecfe5
 SHA512:
-  metadata.gz: 535d2f88c7ff324dbbc0e94dce1e4356125e1b08367c247c48cb083a00dccd997bf3bb73f4534effd8c6797373a588c0e76f6782e60afc36933c21a50b8f9900
-  data.tar.gz: 1e9f92c56e9863d076b0ade0e82917a0e8df50063337d51cda1b9a8df1ea5bdc858b1b153179fd189eaabe313cff8e34213b68daad4488658316cd1dc077eedc
+  metadata.gz: 2e462ba8f51d24c8164ca67804a372d1bea6f6706828801cb2404af02ee183c301e221fef21725d59e013a64652eab06eae84b3e10a427f58ee7e0eed35c3950
+  data.tar.gz: 9594d53c55fc434c92c0601db17c548f473ca62812a0ab57b229d5b9753315198742c83f57a44744b6572def60b4b9659cfa1f93e4d86d66fd6b2f73e0db5234

data/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.8.0] - 2023-08-18
+
+### Added
+
+- Some performance improvements when converting from the C AST to the Ruby AST.
+- Two rust crates have been added: `yarp-sys` and `yarp`. They are as yet unpublished.
+
+### Changed
+
+- Escaped newlines in strings and heredocs are now handled more correctly.
+- Dedenting heredocs that result in empty string nodes will now drop those string nodes from the list.
+- Beginless and endless ranges in conditional expressions now properly form a flip flop node.
+- `%` at the end of files no longer crashes.
+- Location information has been corrected for `if/elsif` chains that have no `else`.
+- `__END__` at the very end of the file was previously parsed as an identifier, but is now correct.
+- **BREAKING**: Nodes that reference `&&=`, `||=`, and other writing operators have been consolidated. Previously, they were separate individual nodes. Now they are a tree with the target being the left-hand side and the value being the right-hand side with a joining `AndWriteNode`, `OrWriteNode`, or `OperatorWriteNode` in the middle. This impacts all of the nodes that match this pattern: `{ClassVariable,Constant,ConstantPath,GlobalVariable,InstanceVariable,LocalVariable}Operator{And,Or,}WriteNode`.
+- **BREAKING**: `BlockParametersNode`, `ClassNode`, `DefNode`, `LambdaNode`, `ModuleNode`, `ParenthesesNode`, and `SingletonClassNode` have had their `statements` field renamed to `body` to give a hint that it might not be a `StatementsNode` (it could also be a `BeginNode`).
+
+## [0.7.0] - 2023-08-14
+
+### Added
+
+- We now have an explicit `FlipFlopNode`. It has the same flags as `RangeNode`.
+- We now have a syntax error when implicit and explicit blocks are passed to a method call.
+- `Node#slice` is now implemented, for retrieving the slice of the source code corresponding to a node.
+- We now support the `utf8-mac` encoding.
+- Predicate methods have been added for nodes that have flags. For example `CallNode#safe_navigation?` and `RangeNode#exclude_end?`.
+- The gem now functions on JRuby and TruffleRuby, thanks to a new FFI backend.
+- Comments are now part of the serialization API.
+
+### Changed
+
+- Autotools has been removed from the build system, so when the gem is installed it will no longer need to go through a configure step.
+- The AST for `foo = *bar` has changed to have an explicit array on the right hand side, rather than a splat node. This is more consistent with how other parsers handle this.
+- **BREAKING**: `RangeNodeFlags` has been renamed to `RangeFlags`.
+- Unary minus on number literals is now parsed as part of the literal, rather than a call to a unary operator. This is more consistent with how other parsers handle this.
+
+## [0.6.0] - 2023-08-09
+
+### Added
+
+- 🎉 Initial release! 🎉
+
+[unreleased]: https://github.com/ruby/yarp/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/ruby/yarp/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/ruby/yarp/compare/v0.6.0...v0.7.0
+[0.6.0]: https://github.com/ruby/yarp/compare/d60531...v0.6.0

data/CONTRIBUTING.md

@@ -10,6 +10,10 @@ The discussions page on the GitHub repository are open. If you have a question o
 
 If you want to contribute code, please first open or contribute to a discussion. A lot of the project is in flux, and we want to make sure that you are contributing to the right place. Once you have a discussion going, you can open a pull request with your changes. We will review your code and get it merged in.
 
+### Ruby Features
+
+Pattern matching and endless method definitions should be avoided as long as the latest TruffleRuby release does not support it.
+
 ## Tests
 
 We could always use more tests! One of the biggest challenges of this project is building up a big test suite. If you want to contribute tests, feel free to open a pull request. These will get merged in as soon as possible.

data/{Makefile.in → Makefile}

@@ -10,10 +10,9 @@ FUZZ_OUTPUT_DIR = $(shell pwd)/fuzz/output
 
 SOEXT := $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
-DEFS := @DEFS@
-CPPFLAGS := @DEFS@ -Iinclude
-CFLAGS := @CFLAGS@ -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
-CC := @CC@
+CPPFLAGS := -Iinclude
+CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
+CC := cc
 
 HEADERS := $(shell find include -name '*.h')
 SOURCES := $(shell find src -name '*.c')
@@ -62,6 +61,8 @@ fuzz-docker-build: fuzz/docker/Dockerfile
 	$(Q) docker build -t yarp/fuzz fuzz/docker/
 
 fuzz-run-%: FORCE fuzz-docker-build
+	$(ECHO) "generating templates"
+	$(Q) bundle exec rake templates
 	$(ECHO) "running $* fuzzer"
 	$(Q) docker run --rm -v $(shell pwd):/yarp yarp/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
 	$(ECHO) "starting AFL++ run"

data/README.md

@@ -8,7 +8,7 @@ The repository contains the infrastructure for both a shared library (librubypar
 
 ```
 .
-├── Makefile.in           configuration to compile the shared library and native tests
+├── Makefile              configuration to compile the shared library and native tests
 ├── Rakefile              configuration to compile the native extension and run the Ruby tests
 ├── bin
 │   ├── lex               runs the lexer on a file or string, prints the tokens, and compares to ripper
@@ -29,6 +29,8 @@ The repository contains the infrastructure for both a shared library (librubypar
 │   ├── yarp              Ruby library files
 │   └── yarp.rb           main entrypoint for the Ruby library
 ├── rakelib               various Rake tasks for the project
+├── rust
+│   └── yarp-sys          FFI binding for Rust
 ├── src
 │   ├── enc               various encoding files
 │   ├── util              various utility files
@@ -45,7 +47,7 @@ The repository contains the infrastructure for both a shared library (librubypar
 To compile the shared library, you will need:
 
 * A C99 compiler
-* autotools (autoconf, automake, libtool)
+* autotools autoconf, automake, libtool)
 * make
 * Ruby 3.3.0-preview1 or later
 
@@ -77,10 +79,11 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio
 * [Configuration](docs/configuration.md)
 * [Design](docs/design.md)
 * [Encoding](docs/encoding.md)
-* [Extension](docs/extension.md)
 * [Fuzzing](docs/fuzzing.md)
 * [Heredocs](docs/heredocs.md)
 * [Mapping](docs/mapping.md)
 * [Ripper](docs/ripper.md)
+* [Ruby API](docs/ruby_api.md)
 * [Serialization](docs/serialization.md)
 * [Testing](docs/testing.md)
+

data/config.yml

@@ -337,7 +337,7 @@ flags:
     values:
       - name: BEGIN_MODIFIER
         comment: "a loop after a begin statement, so the body is executed first before the condition"
-  - name: RangeNodeFlags
+  - name: RangeFlags
     values:
       - name: EXCLUDE_END
         comment: "... operator"
@@ -399,6 +399,19 @@ nodes:
 
           left and right
           ^^^^^^^^^^^^^^
+  - name: AndWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: value
+        type: node
+      - name: operator_loc
+        type: location
+    comment: |
+      Represents the use of the `&&=` operator.
+
+          target &&= value
+          ^^^^^^^^^^^^^^^^
   - name: ArgumentsNode
     child_nodes:
       - name: arguments
@@ -527,7 +540,7 @@ nodes:
       - name: parameters
         type: node?
         kind: BlockParametersNode
-      - name: statements
+      - name: body
         type: node?
       - name: opening_loc
         type: location
@@ -601,7 +614,8 @@ nodes:
         type: node?
         kind: BlockNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: CallNodeFlags
       - name: name
         type: string
     comment: |
@@ -713,7 +727,7 @@ nodes:
         type: location?
       - name: superclass
         type: node?
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -722,47 +736,6 @@ nodes:
 
           class Foo end
           ^^^^^^^^^^^^^
-  - name: ClassVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a class variable.
-
-          @@target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: ClassVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a class variable.
-
-          @@target ||= value
-          ^^^^^^^^^^^^^^^^^^
-  - name: ClassVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a class variable using an operator that isn't `=`.
-
-          @@target += value
-          ^^^^^^^^^^^^^^^^^
   - name: ClassVariableReadNode
     comment: |
       Represents referencing a class variable.
@@ -782,47 +755,6 @@ nodes:
 
           @@foo = 1
           ^^^^^^^^^
-  - name: ConstantOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a constant.
-
-          Target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: ConstantOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a constant.
-
-          Target ||= value
-          ^^^^^^^^^^^^^^^^
-  - name: ConstantOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a constant using an operator that isn't `=`.
-
-          Target += value
-          ^^^^^^^^^^^^^^^
   - name: ConstantPathNode
     child_nodes:
       - name: parent
@@ -836,50 +768,6 @@ nodes:
 
           Foo::Bar
           ^^^^^^^^
-  - name: ConstantPathOperatorAndWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a constant path.
-
-          Parent::Child &&= value
-          ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: ConstantPathOperatorOrWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a constant path.
-
-          Parent::Child ||= value
-          ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: ConstantPathOperatorWriteNode
-    child_nodes:
-      - name: target
-        type: node
-        kind: ConstantPathNode
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a constant path using an operator that isn't `=`.
-
-          Parent::Child += value
-          ^^^^^^^^^^^^^^^^^^^^^^
   - name: ConstantPathWriteNode
     child_nodes:
       - name: target
@@ -928,7 +816,7 @@ nodes:
       - name: parameters
         type: node?
         kind: ParametersNode
-      - name: statements
+      - name: body
         type: node?
       - name: locals
         type: constant[]
@@ -1053,6 +941,22 @@ nodes:
 
           foo in Foo(*bar, baz, *qux)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  - name: FlipFlopNode
+    child_nodes:
+      - name: left
+        type: node?
+      - name: right
+        type: node?
+      - name: operator_loc
+        type: location
+      - name: flags
+        type: flags
+        kind: RangeFlags
+    comment: |
+      Represents the use of the `..` or `...` operators to create flip flops.
+
+          baz if foo .. bar
+                 ^^^^^^^^^^
   - name: FloatNode
     comment: |
       Represents a floating point number literal.
@@ -1106,47 +1010,6 @@ nodes:
 
           super
           ^^^^^
-  - name: GlobalVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a global variable.
-
-          $target &&= value
-          ^^^^^^^^^^^^^^^^^
-  - name: GlobalVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to a global variable.
-
-          $target ||= value
-          ^^^^^^^^^^^^^^^^^
-  - name: GlobalVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to a global variable using an operator that isn't `=`.
-
-          $target += value
-          ^^^^^^^^^^^^^^^^
   - name: GlobalVariableReadNode
     comment: |
       Represents referencing a global variable.
@@ -1246,47 +1109,6 @@ nodes:
 
           case a; in b then c end
                   ^^^^^^^^^^^
-  - name: InstanceVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `&&=` operator for assignment to an instance variable.
-
-          @target &&= value
-          ^^^^^^^^^^^^^^^^^
-  - name: InstanceVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-    comment: |
-      Represents the use of the `||=` operator for assignment to an instance variable.
-
-          @target ||= value
-          ^^^^^^^^^^^^^^^^^
-  - name: InstanceVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: operator
-        type: constant
-    comment: |
-      Represents assigning to an instance variable using an operator that isn't `=`.
-
-          @target += value
-          ^^^^^^^^^^^^^^^^
   - name: InstanceVariableReadNode
     comment: |
       Represents referencing an instance variable.
@@ -1321,7 +1143,8 @@ nodes:
       - name: closing_loc
         type: location
       - name: flags
-        type: uint32
+        type: flags
+        kind: RegularExpressionFlags
     newline: parts
     comment: |
       Represents a regular expression literal that contains interpolation.
@@ -1416,60 +1239,13 @@ nodes:
       - name: parameters
         type: node?
         kind: BlockParametersNode
-      - name: statements
+      - name: body
         type: node?
     comment: |
       Represents using a lambda literal (not the lambda method call).
 
           ->(value) { value * 2 }
           ^^^^^^^^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorAndWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-    comment: |
-      Represents the use of the `&&=` operator for assignment to a local variable.
-
-          target &&= value
-          ^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorOrWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-    comment: |
-      Represents the use of the `||=` operator for assignment to a local variable.
-
-          target ||= value
-          ^^^^^^^^^^^^^^^^
-  - name: LocalVariableOperatorWriteNode
-    child_nodes:
-      - name: name_loc
-        type: location
-      - name: operator_loc
-        type: location
-      - name: value
-        type: node
-      - name: constant_id
-        type: constant
-      - name: operator_id
-        type: constant
-    comment: |
-      Represents assigning to a local variable using an operator that isn't `=`.
-
-          target += value
-          ^^^^^^^^^^^^^^^
   - name: LocalVariableReadNode
     child_nodes:
       - name: constant_id
@@ -1538,7 +1314,7 @@ nodes:
         type: location
       - name: constant_path
         type: node
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -1600,6 +1376,21 @@ nodes:
 
           $1
           ^^
+  - name: OperatorWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: operator_loc
+        type: location
+      - name: operator
+        type: constant
+      - name: value
+        type: node
+    comment: |
+      Represents the use of an operator on a write.
+
+          target += value
+          ^^^^^^^^^^^^^^^
   - name: OptionalParameterNode
     child_nodes:
       - name: constant_id
@@ -1629,6 +1420,19 @@ nodes:
 
           left or right
           ^^^^^^^^^^^^^
+  - name: OrWriteNode
+    child_nodes:
+      - name: target
+        type: node
+      - name: value
+        type: node
+      - name: operator_loc
+        type: location
+    comment: |
+      Represents the use of the `||=` operator.
+
+          target ||= value
+          ^^^^^^^^^^^^^^^^
   - name: ParametersNode
     child_nodes:
       - name: requireds
@@ -1655,7 +1459,7 @@ nodes:
           end
   - name: ParenthesesNode
     child_nodes:
-      - name: statements
+      - name: body
         type: node?
       - name: opening_loc
         type: location
@@ -1744,7 +1548,8 @@ nodes:
       - name: operator_loc
         type: location
       - name: flags
-        type: uint32
+        type: flags
+        kind: RangeFlags
     comment: |
       Represents the use of the `..` or `...` operators.
 
@@ -1779,7 +1584,8 @@ nodes:
       - name: unescaped
         type: string
       - name: flags
-        type: uint32
+        type: flags
+        kind: RegularExpressionFlags
     comment: |
       Represents a regular expression literal with no interpolation.
 
@@ -1896,7 +1702,7 @@ nodes:
         type: location
       - name: expression
         type: node
-      - name: statements
+      - name: body
         type: node?
       - name: end_keyword_loc
         type: location
@@ -1912,7 +1718,6 @@ nodes:
           __ENCODING__
           ^^^^^^^^^^^^
   - name: SourceFileNode
-    is_migrated: true
     child_nodes:
       - name: filepath
         type: string
@@ -2070,7 +1875,8 @@ nodes:
         type: node?
         kind: StatementsNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `until` keyword, either in the block form or the modifier form.
@@ -2090,10 +1896,12 @@ nodes:
         type: node?
         kind: StatementsNode
     comment: |
-      case true
-      when true
-      ^^^^^^^^^
-      end
+      Represents the use of the `when` keyword within a case statement.
+
+          case true
+          when true
+          ^^^^^^^^^
+          end
   - name: WhileNode
     child_nodes:
       - name: keyword_loc
@@ -2104,7 +1912,8 @@ nodes:
         type: node?
         kind: StatementsNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `while` keyword, either in the block form or the modifier form.