yarp 0.8.0 → 0.10.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +48 -1
  3. data/Makefile +5 -1
  4. data/README.md +4 -3
  5. data/config.yml +461 -150
  6. data/docs/configuration.md +1 -0
  7. data/docs/encoding.md +5 -5
  8. data/docs/ruby_api.md +2 -0
  9. data/docs/serialization.md +3 -3
  10. data/docs/testing.md +2 -2
  11. data/ext/yarp/api_node.c +810 -199
  12. data/ext/yarp/extension.c +94 -31
  13. data/ext/yarp/extension.h +2 -2
  14. data/include/yarp/ast.h +653 -150
  15. data/include/yarp/defines.h +2 -1
  16. data/include/yarp/diagnostic.h +3 -3
  17. data/include/yarp/enc/yp_encoding.h +10 -10
  18. data/include/yarp/node.h +10 -0
  19. data/include/yarp/parser.h +19 -19
  20. data/include/yarp/regexp.h +1 -1
  21. data/include/yarp/unescape.h +7 -5
  22. data/include/yarp/util/yp_buffer.h +3 -0
  23. data/include/yarp/util/yp_char.h +16 -16
  24. data/include/yarp/util/yp_constant_pool.h +2 -2
  25. data/include/yarp/util/yp_newline_list.h +7 -4
  26. data/include/yarp/util/yp_string.h +4 -4
  27. data/include/yarp/util/yp_string_list.h +0 -3
  28. data/include/yarp/util/yp_strpbrk.h +1 -1
  29. data/include/yarp/version.h +2 -2
  30. data/include/yarp.h +14 -3
  31. data/lib/yarp/desugar_visitor.rb +204 -0
  32. data/lib/yarp/ffi.rb +27 -1
  33. data/lib/yarp/lex_compat.rb +93 -25
  34. data/lib/yarp/mutation_visitor.rb +683 -0
  35. data/lib/yarp/node.rb +3121 -597
  36. data/lib/yarp/serialize.rb +198 -126
  37. data/lib/yarp.rb +53 -7
  38. data/src/diagnostic.c +1 -1
  39. data/src/enc/yp_big5.c +15 -42
  40. data/src/enc/yp_euc_jp.c +16 -43
  41. data/src/enc/yp_gbk.c +19 -46
  42. data/src/enc/yp_shift_jis.c +16 -43
  43. data/src/enc/yp_tables.c +36 -38
  44. data/src/enc/yp_unicode.c +20 -25
  45. data/src/enc/yp_windows_31j.c +16 -43
  46. data/src/node.c +1444 -836
  47. data/src/prettyprint.c +324 -103
  48. data/src/regexp.c +21 -21
  49. data/src/serialize.c +429 -276
  50. data/src/token_type.c +2 -2
  51. data/src/unescape.c +184 -136
  52. data/src/util/yp_buffer.c +7 -2
  53. data/src/util/yp_char.c +34 -34
  54. data/src/util/yp_constant_pool.c +4 -4
  55. data/src/util/yp_memchr.c +1 -1
  56. data/src/util/yp_newline_list.c +14 -3
  57. data/src/util/yp_string.c +22 -20
  58. data/src/util/yp_string_list.c +0 -6
  59. data/src/util/yp_strncasecmp.c +3 -6
  60. data/src/util/yp_strpbrk.c +8 -8
  61. data/src/yarp.c +1504 -615
  62. data/yarp.gemspec +3 -1
  63. metadata +4 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ef0d177f2be8f0d5ecd0ba54901e3cdf17fd6ec6f44a1b04160082ae5a1cb519
4
- data.tar.gz: 776a4c7b981d4d6f3848bef030a448ea7d402f6ea75c20fcd87d3cbf935ecfe5
3
+ metadata.gz: 7d7fbd6801f30b7ad185ea9884ad6ab4a0b0fd07494dbf96d90a560d7c5223a3
4
+ data.tar.gz: a28e10e86eee28950106cea5be18020349ea5181111f4d45d540322677b5eed4
5
5
  SHA512:
6
- metadata.gz: 2e462ba8f51d24c8164ca67804a372d1bea6f6706828801cb2404af02ee183c301e221fef21725d59e013a64652eab06eae84b3e10a427f58ee7e0eed35c3950
7
- data.tar.gz: 9594d53c55fc434c92c0601db17c548f473ca62812a0ab57b229d5b9753315198742c83f57a44744b6572def60b4b9659cfa1f93e4d86d66fd6b2f73e0db5234
6
+ metadata.gz: c30fdd9590910182706dc3d6f0747ce8c467156ede3c2ed727d8a2efb2aa7f821abd2e5465e80ddfe122ba80c03af23ab1d60c50f7a39348d797106c9c4567d7
7
+ data.tar.gz: 8c877d31c11834e6e0d22b6acab729258cd87ce4a53e026c8b0ff4b92f98f90a0daa57b21811a29d3c4e2387a65b4a3935d165a4311985d40b3511f95c94f280
data/CHANGELOG.md CHANGED
@@ -6,6 +6,51 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.10.0] - 2023-09-01
10
+
11
+ ### Added
12
+
13
+ - `InstanceVariable*Node` and `ClassVariable*Node` objects now have their `name` returning a Symbol. This is because they are now part of the constant pool.
14
+ - `NumberedReferenceReadNode` now has a `number` field, which returns an Integer.
15
+
16
+ ### Changed
17
+
18
+ - **BREAKING**: Various `operator_id` and `constant_id` fields have been renamed to `operator` and `name`, respectively. See [09d0a144](https://github.com/ruby/yarp/commit/09d0a144dfd519c5b5f96f0b6ee95d256e2cb1a6) for details.
19
+ - `%w`, `%W`, `%i`, `%I`, `%q`, and `%Q` literals can now span around the contents of a heredoc.
20
+ - **BREAKING**: All of the public C APIs that accept the source string now accept `const uint8_t *` as opposed to `const char *`.
21
+
22
+ ## [0.9.0] - 2023-08-25
23
+
24
+ ### Added
25
+
26
+ - Regular expressions can now be bound by `\n`, `\r`, and a combination of `\r\n`.
27
+ - Strings delimited by `%`, `%q`, and `%Q` can now be bound by `\n`, `\r`, and a combination of `\r\n`.
28
+ - `IntegerNode#value` now returns the value of the integer as a Ruby `Integer`.
29
+ - `FloatNode#value` now returns the value of the float as a Ruby `Float`.
30
+ - `RationalNode#value` now returns the value of the rational as a Ruby `Rational`.
31
+ - `ImaginaryNode#value` now returns the value of the imaginary as a Ruby `Complex`.
32
+ - `ClassNode#name` is now a string that returns the name of just the class, without the namespace.
33
+ - `ModuleNode#name` is now a string that returns the name of just the module, without the namespace.
34
+ - Regular expressions and strings found after a heredoc declaration but before the heredoc body are now parsed correctly.
35
+ - The serialization API now supports shared strings, which should help reduce the size of the serialized AST.
36
+ - `*Node#copy` is introduced, which returns a copy of the node with the given overrides.
37
+ - `Location#copy` is introduced, which returns a copy of the location with the given overrides.
38
+ - `DesugarVisitor` is introduced, which provides a simpler AST for use in tools that want to process fewer node types.
39
+ - `{ClassVariable,Constant,ConstantPath,GlobalVariable,InstanceVariable,LocalVariable}TargetNode` are introduced. These nodes represent the target of writes in locations where a value cannot be provided, like a multi write or a rescue reference.
40
+ - `UntilNode#closing_loc` and `WhileNode#closing_loc` are now provided.
41
+ - `Location#join` is now provided, which joins two locations together.
42
+ - `YARP::parse_lex` and `YARP::parse_lex_file` are introduced to parse and lex in one result.
43
+
44
+ ### Changed
45
+
46
+ - When there is a magic encoding comment, the encoding of the first token's source string is now properly reencoded.
47
+ - Constants followed by unary `&` are now properly parsed as a call with a passed block argument.
48
+ - Escaping multi-byte characters in a string literal will now properly escape the entire character.
49
+ - `YARP.lex_compat` now has more accurate behavior when a byte-order mark is present in the file.
50
+ - **BREAKING**: `AndWriteNode`, `OrWriteNode`, and `OperatorWriteNode` have been split back up into their `0.7.0` versions.
51
+ - We now properly support spaces between the `encoding` and `=`/`:` in a magic encoding comment.
52
+ - We now properly parse `-> foo: bar do end`.
53
+
9
54
  ## [0.8.0] - 2023-08-18
10
55
 
11
56
  ### Added
@@ -49,7 +94,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
49
94
 
50
95
  - 🎉 Initial release! 🎉
51
96
 
52
- [unreleased]: https://github.com/ruby/yarp/compare/v0.8.0...HEAD
97
+ [unreleased]: https://github.com/ruby/yarp/compare/v0.10.0...HEAD
98
+ [0.10.0]: https://github.com/ruby/yarp/compare/v0.9.0...v0.10.0
99
+ [0.9.0]: https://github.com/ruby/yarp/compare/v0.8.0...v0.9.0
53
100
  [0.8.0]: https://github.com/ruby/yarp/compare/v0.7.0...v0.8.0
54
101
  [0.7.0]: https://github.com/ruby/yarp/compare/v0.6.0...v0.7.0
55
102
  [0.6.0]: https://github.com/ruby/yarp/compare/d60531...v0.6.0
data/Makefile CHANGED
@@ -44,6 +44,7 @@ build/static/%.o: src/%.c Makefile $(HEADERS)
44
44
 
45
45
  build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
46
46
  $(ECHO) "building $* fuzzer"
47
+ $(Q) mkdir -p $(@D)
47
48
  $(ECHO) "building main fuzz binary"
48
49
  $(Q) AFL_HARDEN=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
49
50
  $(ECHO) "building cmplog binary"
@@ -70,10 +71,13 @@ fuzz-run-%: FORCE fuzz-docker-build
70
71
  $(Q) docker run -it --rm -v $(shell pwd):/yarp -v $(FUZZ_OUTPUT_DIR):/fuzz_output yarp/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
71
72
  FORCE:
72
73
 
74
+ fuzz-clean:
75
+ $(Q) rm -f -r fuzz/output
76
+
73
77
  clean:
74
78
  $(Q) rm -f -r build
75
79
 
76
- .PHONY: clean
80
+ .PHONY: clean fuzz-clean
77
81
 
78
82
  all-no-debug: DEBUG_FLAGS := -DNDEBUG=1
79
83
  all-no-debug: OPTFLAGS := -O3
data/README.md CHANGED
@@ -14,7 +14,6 @@ The repository contains the infrastructure for both a shared library (librubypar
14
14
  │   ├── lex runs the lexer on a file or string, prints the tokens, and compares to ripper
15
15
  │   └── parse runs the parser on a file or string and prints the syntax tree
16
16
  ├── config.yml specification for tokens and nodes in the tree
17
- ├── configure.ac configuration to generate the Makefile
18
17
  ├── docs documentation about the project
19
18
  ├── ext
20
19
  │   └── yarp
@@ -30,6 +29,7 @@ The repository contains the infrastructure for both a shared library (librubypar
30
29
  │   └── yarp.rb main entrypoint for the Ruby library
31
30
  ├── rakelib various Rake tasks for the project
32
31
  ├── rust
32
+ │   ├── yarp Rustified crate for the shared library
33
33
  │   └── yarp-sys FFI binding for Rust
34
34
  ├── src
35
35
  │   ├── enc various encoding files
@@ -38,8 +38,9 @@ The repository contains the infrastructure for both a shared library (librubypar
38
38
  ├── templates contains ERB templates generated by templates/template.rb
39
39
  │   └── template.rb generates code from the nodes and tokens configured by config.yml
40
40
  └── test
41
-    ├── fixtures Ruby code used for testing
42
-    └── snapshots snapshots of generated syntax trees corresponding to fixtures
41
+ └── yarp
42
+ ├── fixtures Ruby code used for testing
43
+ └── snapshots snapshots of generated syntax trees corresponding to fixtures
43
44
  ```
44
45
 
45
46
  ## Getting started