yarp 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +48 -1
  3. data/Makefile +5 -1
  4. data/README.md +4 -3
  5. data/config.yml +461 -150
  6. data/docs/configuration.md +1 -0
  7. data/docs/encoding.md +5 -5
  8. data/docs/ruby_api.md +2 -0
  9. data/docs/serialization.md +3 -3
  10. data/docs/testing.md +2 -2
  11. data/ext/yarp/api_node.c +810 -199
  12. data/ext/yarp/extension.c +94 -31
  13. data/ext/yarp/extension.h +2 -2
  14. data/include/yarp/ast.h +653 -150
  15. data/include/yarp/defines.h +2 -1
  16. data/include/yarp/diagnostic.h +3 -3
  17. data/include/yarp/enc/yp_encoding.h +10 -10
  18. data/include/yarp/node.h +10 -0
  19. data/include/yarp/parser.h +19 -19
  20. data/include/yarp/regexp.h +1 -1
  21. data/include/yarp/unescape.h +7 -5
  22. data/include/yarp/util/yp_buffer.h +3 -0
  23. data/include/yarp/util/yp_char.h +16 -16
  24. data/include/yarp/util/yp_constant_pool.h +2 -2
  25. data/include/yarp/util/yp_newline_list.h +7 -4
  26. data/include/yarp/util/yp_string.h +4 -4
  27. data/include/yarp/util/yp_string_list.h +0 -3
  28. data/include/yarp/util/yp_strpbrk.h +1 -1
  29. data/include/yarp/version.h +2 -2
  30. data/include/yarp.h +14 -3
  31. data/lib/yarp/desugar_visitor.rb +204 -0
  32. data/lib/yarp/ffi.rb +27 -1
  33. data/lib/yarp/lex_compat.rb +93 -25
  34. data/lib/yarp/mutation_visitor.rb +683 -0
  35. data/lib/yarp/node.rb +3121 -597
  36. data/lib/yarp/serialize.rb +198 -126
  37. data/lib/yarp.rb +53 -7
  38. data/src/diagnostic.c +1 -1
  39. data/src/enc/yp_big5.c +15 -42
  40. data/src/enc/yp_euc_jp.c +16 -43
  41. data/src/enc/yp_gbk.c +19 -46
  42. data/src/enc/yp_shift_jis.c +16 -43
  43. data/src/enc/yp_tables.c +36 -38
  44. data/src/enc/yp_unicode.c +20 -25
  45. data/src/enc/yp_windows_31j.c +16 -43
  46. data/src/node.c +1444 -836
  47. data/src/prettyprint.c +324 -103
  48. data/src/regexp.c +21 -21
  49. data/src/serialize.c +429 -276
  50. data/src/token_type.c +2 -2
  51. data/src/unescape.c +184 -136
  52. data/src/util/yp_buffer.c +7 -2
  53. data/src/util/yp_char.c +34 -34
  54. data/src/util/yp_constant_pool.c +4 -4
  55. data/src/util/yp_memchr.c +1 -1
  56. data/src/util/yp_newline_list.c +14 -3
  57. data/src/util/yp_string.c +22 -20
  58. data/src/util/yp_string_list.c +0 -6
  59. data/src/util/yp_strncasecmp.c +3 -6
  60. data/src/util/yp_strpbrk.c +8 -8
  61. data/src/yarp.c +1504 -615
  62. data/yarp.gemspec +3 -1
  63. metadata +4 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ef0d177f2be8f0d5ecd0ba54901e3cdf17fd6ec6f44a1b04160082ae5a1cb519
4
- data.tar.gz: 776a4c7b981d4d6f3848bef030a448ea7d402f6ea75c20fcd87d3cbf935ecfe5
3
+ metadata.gz: 7d7fbd6801f30b7ad185ea9884ad6ab4a0b0fd07494dbf96d90a560d7c5223a3
4
+ data.tar.gz: a28e10e86eee28950106cea5be18020349ea5181111f4d45d540322677b5eed4
5
5
  SHA512:
6
- metadata.gz: 2e462ba8f51d24c8164ca67804a372d1bea6f6706828801cb2404af02ee183c301e221fef21725d59e013a64652eab06eae84b3e10a427f58ee7e0eed35c3950
7
- data.tar.gz: 9594d53c55fc434c92c0601db17c548f473ca62812a0ab57b229d5b9753315198742c83f57a44744b6572def60b4b9659cfa1f93e4d86d66fd6b2f73e0db5234
6
+ metadata.gz: c30fdd9590910182706dc3d6f0747ce8c467156ede3c2ed727d8a2efb2aa7f821abd2e5465e80ddfe122ba80c03af23ab1d60c50f7a39348d797106c9c4567d7
7
+ data.tar.gz: 8c877d31c11834e6e0d22b6acab729258cd87ce4a53e026c8b0ff4b92f98f90a0daa57b21811a29d3c4e2387a65b4a3935d165a4311985d40b3511f95c94f280
data/CHANGELOG.md CHANGED
@@ -6,6 +6,51 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.10.0] - 2023-09-01
10
+
11
+ ### Added
12
+
13
+ - `InstanceVariable*Node` and `ClassVariable*Node` objects now have their `name` returning a Symbol. This is because they are now part of the constant pool.
14
+ - `NumberedReferenceReadNode` now has a `number` field, which returns an Integer.
15
+
16
+ ### Changed
17
+
18
+ - **BREAKING**: Various `operator_id` and `constant_id` fields have been renamed to `operator` and `name`, respectively. See [09d0a144](https://github.com/ruby/yarp/commit/09d0a144dfd519c5b5f96f0b6ee95d256e2cb1a6) for details.
19
+ - `%w`, `%W`, `%i`, `%I`, `%q`, and `%Q` literals can now span around the contents of a heredoc.
20
+ - **BREAKING**: All of the public C APIs that accept the source string now accept `const uint8_t *` as opposed to `const char *`.
21
+
22
+ ## [0.9.0] - 2023-08-25
23
+
24
+ ### Added
25
+
26
+ - Regular expressions can now be bound by `\n`, `\r`, and a combination of `\r\n`.
27
+ - Strings delimited by `%`, `%q`, and `%Q` can now be bound by `\n`, `\r`, and a combination of `\r\n`.
28
+ - `IntegerNode#value` now returns the value of the integer as a Ruby `Integer`.
29
+ - `FloatNode#value` now returns the value of the float as a Ruby `Float`.
30
+ - `RationalNode#value` now returns the value of the rational as a Ruby `Rational`.
31
+ - `ImaginaryNode#value` now returns the value of the imaginary as a Ruby `Complex`.
32
+ - `ClassNode#name` is now a string that returns the name of just the class, without the namespace.
33
+ - `ModuleNode#name` is now a string that returns the name of just the module, without the namespace.
34
+ - Regular expressions and strings found after a heredoc declaration but before the heredoc body are now parsed correctly.
35
+ - The serialization API now supports shared strings, which should help reduce the size of the serialized AST.
36
+ - `*Node#copy` is introduced, which returns a copy of the node with the given overrides.
37
+ - `Location#copy` is introduced, which returns a copy of the location with the given overrides.
38
+ - `DesugarVisitor` is introduced, which provides a simpler AST for use in tools that want to process fewer node types.
39
+ - `{ClassVariable,Constant,ConstantPath,GlobalVariable,InstanceVariable,LocalVariable}TargetNode` are introduced. These nodes represent the target of writes in locations where a value cannot be provided, like a multi write or a rescue reference.
40
+ - `UntilNode#closing_loc` and `WhileNode#closing_loc` are now provided.
41
+ - `Location#join` is now provided, which joins two locations together.
42
+ - `YARP::parse_lex` and `YARP::parse_lex_file` are introduced to parse and lex in one result.
43
+
44
+ ### Changed
45
+
46
+ - When there is a magic encoding comment, the encoding of the first token's source string is now properly reencoded.
47
+ - Constants followed by unary `&` are now properly parsed as a call with a passed block argument.
48
+ - Escaping multi-byte characters in a string literal will now properly escape the entire character.
49
+ - `YARP.lex_compat` now has more accurate behavior when a byte-order mark is present in the file.
50
+ - **BREAKING**: `AndWriteNode`, `OrWriteNode`, and `OperatorWriteNode` have been split back up into their `0.7.0` versions.
51
+ - We now properly support spaces between the `encoding` and `=`/`:` in a magic encoding comment.
52
+ - We now properly parse `-> foo: bar do end`.
53
+
9
54
  ## [0.8.0] - 2023-08-18
10
55
 
11
56
  ### Added
@@ -49,7 +94,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
49
94
 
50
95
  - 🎉 Initial release! 🎉
51
96
 
52
- [unreleased]: https://github.com/ruby/yarp/compare/v0.8.0...HEAD
97
+ [unreleased]: https://github.com/ruby/yarp/compare/v0.10.0...HEAD
98
+ [0.10.0]: https://github.com/ruby/yarp/compare/v0.9.0...v0.10.0
99
+ [0.9.0]: https://github.com/ruby/yarp/compare/v0.8.0...v0.9.0
53
100
  [0.8.0]: https://github.com/ruby/yarp/compare/v0.7.0...v0.8.0
54
101
  [0.7.0]: https://github.com/ruby/yarp/compare/v0.6.0...v0.7.0
55
102
  [0.6.0]: https://github.com/ruby/yarp/compare/d60531...v0.6.0
data/Makefile CHANGED
@@ -44,6 +44,7 @@ build/static/%.o: src/%.c Makefile $(HEADERS)
44
44
 
45
45
  build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
46
46
  $(ECHO) "building $* fuzzer"
47
+ $(Q) mkdir -p $(@D)
47
48
  $(ECHO) "building main fuzz binary"
48
49
  $(Q) AFL_HARDEN=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
49
50
  $(ECHO) "building cmplog binary"
@@ -70,10 +71,13 @@ fuzz-run-%: FORCE fuzz-docker-build
70
71
  $(Q) docker run -it --rm -v $(shell pwd):/yarp -v $(FUZZ_OUTPUT_DIR):/fuzz_output yarp/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
71
72
  FORCE:
72
73
 
74
+ fuzz-clean:
75
+ $(Q) rm -f -r fuzz/output
76
+
73
77
  clean:
74
78
  $(Q) rm -f -r build
75
79
 
76
- .PHONY: clean
80
+ .PHONY: clean fuzz-clean
77
81
 
78
82
  all-no-debug: DEBUG_FLAGS := -DNDEBUG=1
79
83
  all-no-debug: OPTFLAGS := -O3
data/README.md CHANGED
@@ -14,7 +14,6 @@ The repository contains the infrastructure for both a shared library (librubypar
14
14
  │   ├── lex runs the lexer on a file or string, prints the tokens, and compares to ripper
15
15
  │   └── parse runs the parser on a file or string and prints the syntax tree
16
16
  ├── config.yml specification for tokens and nodes in the tree
17
- ├── configure.ac configuration to generate the Makefile
18
17
  ├── docs documentation about the project
19
18
  ├── ext
20
19
  │   └── yarp
@@ -30,6 +29,7 @@ The repository contains the infrastructure for both a shared library (librubypar
30
29
  │   └── yarp.rb main entrypoint for the Ruby library
31
30
  ├── rakelib various Rake tasks for the project
32
31
  ├── rust
32
+ │   ├── yarp Rustified crate for the shared library
33
33
  │   └── yarp-sys FFI binding for Rust
34
34
  ├── src
35
35
  │   ├── enc various encoding files
@@ -38,8 +38,9 @@ The repository contains the infrastructure for both a shared library (librubypar
38
38
  ├── templates contains ERB templates generated by templates/template.rb
39
39
  │   └── template.rb generates code from the nodes and tokens configured by config.yml
40
40
  └── test
41
-    ├── fixtures Ruby code used for testing
42
-    └── snapshots snapshots of generated syntax trees corresponding to fixtures
41
+ └── yarp
42
+ ├── fixtures Ruby code used for testing
43
+ └── snapshots snapshots of generated syntax trees corresponding to fixtures
43
44
  ```
44
45
 
45
46
  ## Getting started