yarp 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51ee2a88319b74a9937ab37fbaa9bb71586caf26a8f31282c2d2431735a484b6
-  data.tar.gz: efe7ac9662cd199e3ba0c8a00f913912a2830843bc0c87cdfc3b258d724eafd8
+  metadata.gz: 81d6e0266ee0d13563faafa47a0c0d09c5feabe870e2324f32564a7f68299dff
+  data.tar.gz: 771e9fd99caaf99cf7493d897cf9fdd5f93824e90bf09c22a1e5b5ec0e1a4bcf
 SHA512:
-  metadata.gz: 535d2f88c7ff324dbbc0e94dce1e4356125e1b08367c247c48cb083a00dccd997bf3bb73f4534effd8c6797373a588c0e76f6782e60afc36933c21a50b8f9900
-  data.tar.gz: 1e9f92c56e9863d076b0ade0e82917a0e8df50063337d51cda1b9a8df1ea5bdc858b1b153179fd189eaabe313cff8e34213b68daad4488658316cd1dc077eedc
+  metadata.gz: 16525d9ed87d00d9fd94af5cab5f99b8278ac4a81c7117195e80d3cd3cc6b12358fd0f2255cbe7df15ef1adef609719371bc3a210d8a2273d6e34fbdcb4b1a58
+  data.tar.gz: efbf02ccd9a7cb6a13b6ba56dc8fc3fc6e425d7d779583614fc9c24c74216fe3d89eadc83999f197a7227bfd90ca8ba9446c1b71c128e4874609f44f0869eb72

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.7.0] - 2023-08-14
+
+### Added
+
+- We now have an explicit `FlipFlopNode`. It has the same flags as `RangeNode`.
+- We now have a syntax error when implicit and explicit blocks are passed to a method call.
+- `Node#slice` is now implemented, for retrieving the slice of the source code corresponding to a node.
+- We now support the `utf8-mac` encoding.
+- Predicate methods have been added for nodes that have flags. For example `CallNode#safe_navigation?` and `RangeNode#exclude_end?`.
+- The gem now functions on JRuby and TruffleRuby, thanks to a new FFI backend.
+- Comments are now part of the serialization API.
+
+### Changed
+
+- Autotools has been removed from the build system, so when the gem is installed it will no longer need to go through a configure step.
+- The AST for `foo = *bar` has changed to have an explicit array on the right hand side, rather than a splat node. This is more consistent with how other parsers handle this.
+- `RangeNodeFlags` has been renamed to `RangeFlags`.
+- Unary minus on number literals is now parsed as part of the literal, rather than a call to a unary operator. This is more consistent with how other parsers handle this.
+
+## [0.6.0] - 2023-08-09
+
+### Added
+
+- 🎉 Initial release! 🎉
+
+[unreleased]: https://github.com/ruby/yarp/compare/v0.7.0...HEAD
+[0.7.0]: https://github.com/ruby/yarp/compare/v0.6.0...v0.7.0
+[0.6.0]: https://github.com/ruby/yarp/compare/d60531...v0.6.0

data/CONTRIBUTING.md

@@ -10,6 +10,10 @@ The discussions page on the GitHub repository are open. If you have a question o
 
 If you want to contribute code, please first open or contribute to a discussion. A lot of the project is in flux, and we want to make sure that you are contributing to the right place. Once you have a discussion going, you can open a pull request with your changes. We will review your code and get it merged in.
 
+### Ruby Features
+
+Pattern matching and endless method definitions should be avoided as long as the latest TruffleRuby release does not support it.
+
 ## Tests
 
 We could always use more tests! One of the biggest challenges of this project is building up a big test suite. If you want to contribute tests, feel free to open a pull request. These will get merged in as soon as possible.

data/{Makefile.in → Makefile}

@@ -10,10 +10,9 @@ FUZZ_OUTPUT_DIR = $(shell pwd)/fuzz/output
 
 SOEXT := $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
-DEFS := @DEFS@
-CPPFLAGS := @DEFS@ -Iinclude
-CFLAGS := @CFLAGS@ -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
-CC := @CC@
+CPPFLAGS := -Iinclude
+CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
+CC := cc
 
 HEADERS := $(shell find include -name '*.h')
 SOURCES := $(shell find src -name '*.c')

data/README.md

@@ -8,7 +8,7 @@ The repository contains the infrastructure for both a shared library (librubypar
 
 ```
 .
-├── Makefile.in           configuration to compile the shared library and native tests
+├── Makefile              configuration to compile the shared library and native tests
 ├── Rakefile              configuration to compile the native extension and run the Ruby tests
 ├── bin
 │   ├── lex               runs the lexer on a file or string, prints the tokens, and compares to ripper

data/config.yml

@@ -337,7 +337,7 @@ flags:
     values:
       - name: BEGIN_MODIFIER
         comment: "a loop after a begin statement, so the body is executed first before the condition"
-  - name: RangeNodeFlags
+  - name: RangeFlags
     values:
       - name: EXCLUDE_END
         comment: "... operator"
@@ -601,7 +601,8 @@ nodes:
         type: node?
         kind: BlockNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: CallNodeFlags
       - name: name
         type: string
     comment: |
@@ -1053,6 +1054,22 @@ nodes:
 
           foo in Foo(*bar, baz, *qux)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  - name: FlipFlopNode
+    child_nodes:
+      - name: left
+        type: node?
+      - name: right
+        type: node?
+      - name: operator_loc
+        type: location
+      - name: flags
+        type: flags
+        kind: RangeFlags
+    comment: |
+      Represents the use of the `..` or `...` operators to create flip flops.
+
+          baz if foo .. bar
+                 ^^^^^^^^^^
   - name: FloatNode
     comment: |
       Represents a floating point number literal.
@@ -1321,7 +1338,8 @@ nodes:
       - name: closing_loc
         type: location
       - name: flags
-        type: uint32
+        type: flags
+        kind: RegularExpressionFlags
     newline: parts
     comment: |
       Represents a regular expression literal that contains interpolation.
@@ -1744,7 +1762,8 @@ nodes:
       - name: operator_loc
         type: location
       - name: flags
-        type: uint32
+        type: flags
+        kind: RangeFlags
     comment: |
       Represents the use of the `..` or `...` operators.
 
@@ -1779,7 +1798,8 @@ nodes:
       - name: unescaped
         type: string
       - name: flags
-        type: uint32
+        type: flags
+        kind: RegularExpressionFlags
     comment: |
       Represents a regular expression literal with no interpolation.
 
@@ -2070,7 +2090,8 @@ nodes:
         type: node?
         kind: StatementsNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `until` keyword, either in the block form or the modifier form.
@@ -2104,7 +2125,8 @@ nodes:
         type: node?
         kind: StatementsNode
       - name: flags
-        type: uint32
+        type: flags
+        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `while` keyword, either in the block form or the modifier form.

data/docs/build_system.md

@@ -16,8 +16,6 @@ The main solution for the second point seems a Makefile, otherwise many of the u
 ## General Design
 
 1. Templates are generated by `templates/template.rb`
-2. `autoconf` creates `./configure` and `autoheader` creates `config.h.in` (both files are platform-independent)
-3. `./configure` creates `include/yarp/config.h` (which contains `HAVE_*` macros, platform-specific) and the `Makefile`
 4. The `Makefile` compiles both `librubyparser.a` and `librubyparser.{so,dylib,dll}` from the `src/**/*.c` and `include/**/*.h` files
 5. The `Rakefile` `:compile` task ensures the above prerequisites are done, then calls `make`,
   and uses `Rake::ExtensionTask` to compile the C extension (using its `extconf.rb`), which uses `librubyparser.a`
@@ -36,14 +34,13 @@ loaded per process (i.e., at most one version of the yarp *gem* loaded in a proc
 
 ### Building the yarp gem by `gem install/bundle install`
 
-The gem contains the pre-generated templates, as well as `configure` and `config.h.in`
+The gem contains the pre-generated templates.
 When installing the gem, `extconf.rb` is used and that:
-* runs `./configure` which creates the `Makefile` and `include/yarp/config.h`
 * runs `make build/librubyparser.a`
 * compiles the C extension with mkmf
 
 When installing the gem on JRuby and TruffleRuby, no C extension is built, so instead of the last step,
-there is Ruby code using Fiddle which uses `librubyparser.{so,dylib,dll}`
+there is Ruby code using FFI which uses `librubyparser.{so,dylib,dll}`
 to implement the same methods as the C extension, but using serialization instead of many native calls/accesses
 (JRuby does not support C extensions, serialization is faster on TruffleRuby than the C extension).
 
@@ -51,7 +48,6 @@ to implement the same methods as the C extension, but using serialization instea
 
 The same as above, except the `extconf.rb` additionally runs first:
 * `templates/template.rb` to generate the templates
-* `autoconf` and `autoheader` to generate `configure` and `config.h.in`
 
 Because of course those files are not part of the git repository.
 
@@ -61,21 +57,14 @@ Because of course those files are not part of the git repository.
 
 The script generates the templates when importing.
 
-`include/yarp/config.h` is replaced by `#include "ruby/config.h"`.
-It is assumed that CRuby's `./configure` is a superset of YARP's configure checks.
-
-YARP's `autotools` is not used at all in CRuby and in fact YARP's `Makefile` is not used either.
-Instead, CRuby's `autotools` setup is used, and `CRuby`'s Makefiles are used.
+YARP's `Makefile` is not used at all in CRuby. Instead, CRuby's `Makefile` is used.
 
 ### Building YARP as part of TruffleRuby
 
 [This script](https://github.com/oracle/truffleruby/blob/master/tool/import-yarp.sh) imports YARP sources in TruffleRuby.
 The script generates the templates when importing.
-It also generates `configure` and `config.h.in` (to avoid needing `autotools` on every machine building TruffleRuby).
 
-Then when `mx build` builds TruffleRuby and the `yarp` mx project inside, it:
-* runs `./configure`
-* runs `make`
+Then when `mx build` builds TruffleRuby and the `yarp` mx project inside, it runs `make`.
 
 Then the `yarp bindings` mx project is built, which contains the [bindings](https://github.com/oracle/truffleruby/blob/master/src/main/c/yarp_bindings/src/yarp_bindings.c)
 and links to `librubyparser.a` (to avoid exporting symbols, so no conflict when installing the yarp gem).

data/docs/building.md

@@ -1,6 +1,7 @@
 # Building
 
 The following describes how to build YARP from source.
+This comes directly from the [Makefile](../Makefile).
 
 ## Common
 
@@ -13,11 +14,6 @@ The following flags should be used to compile YARP:
 * `-Werror` - Treat warnings as errors
 * `-fvisibility=hidden` - Hide all symbols by default
 
-The following flags can be used to compile YARP:
-
-* `-DHAVE_MMAP` - Should be passed if the system has the `mmap` function
-* `-DHAVE_SNPRINTF` - Should be passed if the system has the `snprintf` function
-
 ## Shared
 
 If you want to build YARP as a shared library and link against it, you should compile with:

data/docs/encoding.md

@@ -39,6 +39,7 @@ The key of the comment can be either "encoding" or "coding". The value of the co
 * `sjis`
 * `us-ascii`
 * `utf-8`
+* `utf8-mac`
 * `windows-31j`
 * `windows-1251`
 * `windows-1252`

data/docs/{extension.md → ruby_api.md}

@@ -1,6 +1,6 @@
-# Extension
+# Ruby API
 
-Part of this parser project is a native extension that provides a Ruby API that wraps calls to the C API. This allows you to invoke parsing from Ruby code. It also provides a Ruby API for accessing the syntax tree.
+The `yarp` gem provides a Ruby API for accessing the syntax tree.
 
 For the most part, the API for accessing the tree mirrors that found in the [Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree) project. This means:
 
@@ -8,7 +8,9 @@ For the most part, the API for accessing the tree mirrors that found in the [Syn
 * Nodes in the tree respond to named methods for accessing their children as well as `#child_nodes`
 * Nodes respond to the pattern matching interfaces `#deconstruct` and `#deconstruct_keys`
 
-Every entry in `config.yml` will generate a Ruby class as well as the code that builds the nodes themselves. Creating a syntax tree involves calling one of the class methods on the `YARP` module. The full API is documented below.
+Every entry in `config.yml` will generate a Ruby class as well as the code that builds the nodes themselves.
+Creating a syntax tree involves calling one of the class methods on the `YARP` module.
+The full API is documented below.
 
 ## API
 
@@ -18,3 +20,4 @@ Every entry in `config.yml` will generate a Ruby class as well as the code that
 * `YARP.lex_file(filepath)` - parse the tokens corresponding to the given source file and return them as an array within a parse result
 * `YARP.parse(source)` - parse the syntax tree corresponding to the given source string and return it within a parse result
 * `YARP.parse_file(filepath)` - parse the syntax tree corresponding to the given source file and return it within a parse result
+* `YARP.load(source, serialized)` - load the serialized syntax tree using the source as a reference into a syntax tree

data/docs/serialization.md

@@ -1,10 +1,58 @@
 # Serialization
 
-YARP ships with the ability to serialize a syntax tree to a single string. The string can then be deserialized back into a syntax tree using a language other than C. This is useful for using the parsing logic in other tools without having to write a parser in that language. The syntax tree still requires a copy of the original source, as for the most part it just contains byte offsets into the source string.
+YARP ships with the ability to serialize a syntax tree to a single string.
+The string can then be deserialized back into a syntax tree using a language other than C.
+This is useful for using the parsing logic in other tools without having to write a parser in that language.
+The syntax tree still requires a copy of the original source, as for the most part it just contains byte offsets into the source string.
+
+## Types
+
+Let us define some simple types for readability.
+
+### varint
+
+A variable-length integer with the value fitting in `uint32_t` using between 1 and 5 bytes, using the [LEB128](https://en.wikipedia.org/wiki/LEB128) encoding.
+This drastically cuts down on the size of the serialized string, especially when the source file is large.
+
+### string
+
+| # bytes | field |
+| --- | --- |
+| varint | the length of the string in bytes |
+| ... | the string bytes |
+
+### location
+
+| # bytes | field |
+| --- | --- |
+| varint | byte offset into the source string where this location begins |
+| varint | length of the location in bytes in the source string |
+
+### comment
+
+The comment type is one of:
+* 0=`INLINE` (`# comment`)
+* 1=`EMBEDDED_DOCUMENT` (`=begin`/`=end`)
+* 2=`__END__` (after `__END__`)
+
+| # bytes | field |
+| --- | --- |
+| `1` | comment type |
+| location | the location in the source of this comment |
+
+### diagnostic
+
+| # bytes | field |
+| --- | --- |
+| string | diagnostic message (ASCII-only characters) |
+| location | the location in the source this diagnostic applies to |
 
 ## Structure
 
-The serialized string representing the syntax tree is composed of three parts: the header, the body, and the constant pool. The header contains information like the version of YARP that serialized the tree. The body contains the actual nodes in the tree. The constant pool contains constants that were interned while parsing.
+The serialized string representing the syntax tree is composed of three parts: the header, the body, and the constant pool.
+The header contains information like the version of YARP that serialized the tree.
+The body contains the actual nodes in the tree.
+The constant pool contains constants that were interned while parsing.
 
 The header is structured like the following table:
 
@@ -14,32 +62,28 @@ The header is structured like the following table:
 | `1` | major version number |
 | `1` | minor version number |
 | `1` | patch version number |
-| varint | the length of the encoding name |
 | string | the encoding name |
+| varint | number of comments |
+| comment* | comments |
 | varint | number of errors |
-| varint | byte length of error |
-| string | error string, as byte[] in source encoding |
-| varint | location in the source code - start |
-| varint | location in the source code - length |
-| ... | more errors |
+| diagnostic* | errors |
 | varint | number of warnings |
-| varint | byte length of warning |
-| string | warning string, as byte[] in source encoding |
-| varint | location in the source code - start |
-| varint | location in the source code - length |
-| ... | more warnings |
+| diagnostic* | warnings |
 | `4` | content pool offset |
 | varint | content pool size |
 
-After the header comes the body of the serialized string. The body consistents of a sequence of nodes that is built using a prefix traversal order of the syntax tree. Each node is structured like the following table:
+After the header comes the body of the serialized string.
+The body consistents of a sequence of nodes that is built using a prefix traversal order of the syntax tree.
+Each node is structured like the following table:
 
 | # bytes | field |
 | --- | --- |
 | `1` | node type |
-| varint | byte offset into the source string where this node begins |
-| varint | length of the node in bytes in the source string |
+| location | node location |
 
-Each node's child is then appended to the serialized string. The child node types can be determined by referencing `config.yml`. Depending on the type of child node, it could take a couple of different forms, described below:
+Each node's child is then appended to the serialized string.
+The child node types can be determined by referencing `config.yml`.
+Depending on the type of child node, it could take a couple of different forms, described below:
 
 * `node` - A child node that is a node itself. This is structured just as like parent node.
 * `node?` - A child node that is optionally present. If the node is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just as like parent node.
@@ -52,7 +96,10 @@ Each node's child is then appended to the serialized string. The child node type
 * `location[]` - A child node that is an array of locations. This is structured as a `4` byte length, followed by the locations themselves.
 * `uint32` - A child node that is a 32-bit unsigned integer. This is structured as a variable-length integer.
 
-After the syntax tree, the content pool is serialized. This is a list of constants that were referenced from within the tree. The content pool begins at the offset specified in the header. Each constant is structured as:
+After the syntax tree, the content pool is serialized.
+This is a list of constants that were referenced from within the tree.
+The content pool begins at the offset specified in the header.
+Each constant is structured as:
 
 | # bytes | field |
 | --- | --- |
@@ -61,10 +108,6 @@ After the syntax tree, the content pool is serialized. This is a list of constan
 
 At the end of the serialization, the buffer is null terminated.
 
-## Variable-length integers
-
-Variable-length integers are used throughout the serialized format, using the [LEB128](https://en.wikipedia.org/wiki/LEB128) encoding. This drastically cuts down on the size of the serialized string, especially when the source file is large.
-
 ## APIs
 
 The relevant APIs and struct definitions are listed below:
@@ -105,7 +148,10 @@ serialize(const char *source, size_t length) {
 }
 ```
 
-The final argument to `yp_parse_serialize` controls the metadata of the source. This includes the filepath that the source is associated with, and any nested local variables scopes that are necessary to properly parse the file (in the case of parsing an `eval`). The metadata is a serialized format itself, and is structured as follows:
+The final argument to `yp_parse_serialize` controls the metadata of the source.
+This includes the filepath that the source is associated with, and any nested local variables scopes that are necessary to properly parse the file (in the case of parsing an `eval`).
+Note that no `varint` are used here to make it easier to produce the metadata for the caller, and also serialized size is less important here.
+The metadata is a serialized format itself, and is structured as follows:
 
 | # bytes | field |
 | --- | --- |
@@ -127,4 +173,5 @@ Each local variable within each scope is encoded as:
 | `4` | the size of the local variable name |
 | | the local variable name |
 
-The metadata can be `NULL` (as seen in the example above). If it is not null, then a minimal metadata string would be `"\0\0\0\0\0\0\0\0"` which would use 4 bytes to indicate an empty filepath string and 4 bytes to indicate that there were no local variable scopes.
+The metadata can be `NULL` (as seen in the example above).
+If it is not null, then a minimal metadata string would be `"\0\0\0\0\0\0\0\0"` which would use 4 bytes to indicate an empty filepath string and 4 bytes to indicate that there were no local variable scopes.

data/ext/yarp/api_node.c

@@ -59,6 +59,7 @@ static VALUE rb_cYARPEmbeddedVariableNode;
 static VALUE rb_cYARPEnsureNode;
 static VALUE rb_cYARPFalseNode;
 static VALUE rb_cYARPFindPatternNode;
+static VALUE rb_cYARPFlipFlopNode;
 static VALUE rb_cYARPFloatNode;
 static VALUE rb_cYARPForNode;
 static VALUE rb_cYARPForwardingArgumentsNode;
@@ -519,6 +520,13 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     yp_node_stack_push(&node_stack, (yp_node_t *) cast->right);
                     break;
                 }
+#line 111 "api_node.c.erb"
+                case YP_NODE_FLIP_FLOP_NODE: {
+                    yp_flip_flop_node_t *cast = (yp_flip_flop_node_t *) node;
+                    yp_node_stack_push(&node_stack, (yp_node_t *) cast->left);
+                    yp_node_stack_push(&node_stack, (yp_node_t *) cast->right);
+                    break;
+                }
 #line 111 "api_node.c.erb"
                 case YP_NODE_FOR_NODE: {
                     yp_for_node_t *cast = (yp_for_node_t *) node;
@@ -1280,7 +1288,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[6] = rb_ary_pop(value_stack);
 
                     // flags
-                    argv[7] = ULONG2NUM(cast->flags);
+                    argv[7] = ULONG2NUM(node->flags >> 1);
 
                     // name
                     argv[8] = yp_string_new(&cast->name, encoding);
@@ -1916,6 +1924,29 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     rb_ary_push(value_stack, rb_class_new_instance(7, argv, rb_cYARPFindPatternNode));
                     break;
                 }
+#line 137 "api_node.c.erb"
+                case YP_NODE_FLIP_FLOP_NODE: {
+                    yp_flip_flop_node_t *cast = (yp_flip_flop_node_t *) node;
+                    VALUE argv[5];
+
+                    // left
+                    argv[0] = rb_ary_pop(value_stack);
+
+                    // right
+                    argv[1] = rb_ary_pop(value_stack);
+
+                    // operator_loc
+                    argv[2] = yp_location_new(parser, cast->operator_loc.start, cast->operator_loc.end, source);
+
+                    // flags
+                    argv[3] = ULONG2NUM(node->flags >> 1);
+
+                    // location
+                    argv[4] = yp_location_new(parser, node->location.start, node->location.end, source);
+
+                    rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cYARPFlipFlopNode));
+                    break;
+                }
 #line 137 "api_node.c.erb"
                 case YP_NODE_FLOAT_NODE: {
                     VALUE argv[1];
@@ -2319,7 +2350,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[2] = yp_location_new(parser, cast->closing_loc.start, cast->closing_loc.end, source);
 
                     // flags
-                    argv[3] = ULONG2NUM(cast->flags);
+                    argv[3] = ULONG2NUM(node->flags >> 1);
 
                     // location
                     argv[4] = yp_location_new(parser, node->location.start, node->location.end, source);
@@ -2978,7 +3009,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[2] = yp_location_new(parser, cast->operator_loc.start, cast->operator_loc.end, source);
 
                     // flags
-                    argv[3] = ULONG2NUM(cast->flags);
+                    argv[3] = ULONG2NUM(node->flags >> 1);
 
                     // location
                     argv[4] = yp_location_new(parser, node->location.start, node->location.end, source);
@@ -3027,7 +3058,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[3] = yp_string_new(&cast->unescaped, encoding);
 
                     // flags
-                    argv[4] = ULONG2NUM(cast->flags);
+                    argv[4] = ULONG2NUM(node->flags >> 1);
 
                     // location
                     argv[5] = yp_location_new(parser, node->location.start, node->location.end, source);
@@ -3437,7 +3468,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[2] = rb_ary_pop(value_stack);
 
                     // flags
-                    argv[3] = ULONG2NUM(cast->flags);
+                    argv[3] = ULONG2NUM(node->flags >> 1);
 
                     // location
                     argv[4] = yp_location_new(parser, node->location.start, node->location.end, source);
@@ -3483,7 +3514,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                     argv[2] = rb_ary_pop(value_stack);
 
                     // flags
-                    argv[3] = ULONG2NUM(cast->flags);
+                    argv[3] = ULONG2NUM(node->flags >> 1);
 
                     // location
                     argv[4] = yp_location_new(parser, node->location.start, node->location.end, source);
@@ -3595,6 +3626,7 @@ Init_yarp_api_node(void) {
     rb_cYARPEnsureNode = rb_define_class_under(rb_cYARP, "EnsureNode", rb_cYARPNode);
     rb_cYARPFalseNode = rb_define_class_under(rb_cYARP, "FalseNode", rb_cYARPNode);
     rb_cYARPFindPatternNode = rb_define_class_under(rb_cYARP, "FindPatternNode", rb_cYARPNode);
+    rb_cYARPFlipFlopNode = rb_define_class_under(rb_cYARP, "FlipFlopNode", rb_cYARPNode);
     rb_cYARPFloatNode = rb_define_class_under(rb_cYARP, "FloatNode", rb_cYARPNode);
     rb_cYARPForNode = rb_define_class_under(rb_cYARP, "ForNode", rb_cYARPNode);
     rb_cYARPForwardingArgumentsNode = rb_define_class_under(rb_cYARP, "ForwardingArgumentsNode", rb_cYARPNode);

data/ext/yarp/extconf.rb

@@ -1,13 +1,21 @@
 # frozen_string_literal: true
 
-require "mkmf"
 require "rbconfig"
-require "rake"
 
 module Yarp
   module ExtConf
     class << self
       def configure
+        unless RUBY_ENGINE == "ruby"
+          # On non-CRuby we only need the shared library, so build only that and not the C extension.
+          # We also avoid `require "mkmf"` as that prepends the LLVM toolchain to PATH on TruffleRuby,
+          # but we want to use the native toolchain here since librubyparser is run natively.
+          build_shared_rubyparser
+          File.write("Makefile", "all install clean:\n\t@#{RbConfig::CONFIG["NULLCMD"]}\n")
+          return
+        end
+
+        require "mkmf"
         configure_c_extension
         configure_rubyparser
 
@@ -35,7 +43,7 @@ module Yarp
           end
           $LOCAL_LIBS << " #{static_archive_path}"
         else
-          shared_library_path = File.join(build_dir, "librubyparser.#{RbConfig::CONFIG["DLEXT"]}")
+          shared_library_path = File.join(build_dir, "librubyparser.#{RbConfig::CONFIG["SOEXT"]}")
           unless File.exist?(shared_library_path)
             build_shared_rubyparser
           end
@@ -62,15 +70,12 @@ module Yarp
 
       def build_target_rubyparser(target)
         Dir.chdir(root_dir) do
-          if !File.exist?("configure") && Dir.exist?(".git")
+          if !File.exist?("include/yarp/ast.h") && Dir.exist?(".git")
             # this block only exists to support building the gem from a "git" source,
             # normally we package up the configure and other files in the gem itself
-            Rake.sh("autoconf")
-            Rake.sh("autoheader")
-            Rake.sh("templates/template.rb")
+            system("templates/template.rb", exception: true)
           end
-          Rake.sh("sh", "configure") # explicit "sh" for Windows where shebangs are not supported
-          Rake.sh("make", target)
+          system("make", target, exception: true)
         end
       end
 
@@ -123,7 +128,7 @@ module Yarp
   end
 end
 
-if arg_config("--help")
+if ARGV.delete("--help")
   Yarp::ExtConf.print_help
   exit!(0)
 end

data/ext/yarp/extension.c

@@ -522,6 +522,8 @@ Init_yarp(void) {
     // in yarp.h.
     rb_define_const(rb_cYARP, "VERSION", rb_str_new2(EXPECTED_YARP_VERSION));
 
+    rb_define_const(rb_cYARP, "BACKEND", ID2SYM(rb_intern("CExtension")));
+
     // First, the functions that have to do with lexing and parsing.
     rb_define_singleton_method(rb_cYARP, "dump", dump, -1);
     rb_define_singleton_method(rb_cYARP, "dump_file", dump_file, 1);

data/ext/yarp/extension.h

@@ -5,7 +5,7 @@
 #include <ruby/encoding.h>
 #include "yarp.h"
 
-#define EXPECTED_YARP_VERSION "0.6.0"
+#define EXPECTED_YARP_VERSION "0.7.0"
 
 VALUE yp_source_new(yp_parser_t *parser);
 VALUE yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALUE source);