prism 1.9.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -40
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +49 -20
  6. data/README.md +38 -98
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +9 -16
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +39 -28
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1527 -1467
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +759 -418
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +45 -0
  23. data/include/prism/arena.hpp +42 -0
  24. data/include/prism/ast.h +2738 -611
  25. data/include/prism/ast.hpp +641 -0
  26. data/include/prism/buffer.h +60 -0
  27. data/include/prism/buffer.hpp +55 -0
  28. data/include/prism/comments.h +72 -0
  29. data/include/prism/compiler/accel.h +26 -0
  30. data/include/prism/compiler/align.h +36 -0
  31. data/include/prism/compiler/assume.h +25 -0
  32. data/include/prism/compiler/branch.h +26 -0
  33. data/include/prism/compiler/bswap.h +46 -0
  34. data/include/prism/compiler/cplusplus.hpp +16 -0
  35. data/include/prism/compiler/endianness.h +34 -0
  36. data/include/prism/compiler/exceptions.hpp +17 -0
  37. data/include/prism/compiler/exported.h +24 -0
  38. data/include/prism/compiler/fallthrough.h +22 -0
  39. data/include/prism/compiler/filesystem.h +32 -0
  40. data/include/prism/compiler/flex_array.h +19 -0
  41. data/include/prism/compiler/force_inline.h +21 -0
  42. data/include/prism/compiler/format.h +27 -0
  43. data/include/prism/compiler/inline.h +17 -0
  44. data/include/prism/compiler/lifetimebound.hpp +25 -0
  45. data/include/prism/compiler/nodiscard.h +22 -0
  46. data/include/prism/compiler/nonnull.h +18 -0
  47. data/include/prism/compiler/unused.h +18 -0
  48. data/include/prism/constant_pool.h +106 -0
  49. data/include/prism/diagnostic.h +55 -402
  50. data/include/prism/errors_format.h +61 -0
  51. data/include/prism/excludes.h +29 -0
  52. data/include/prism/integer.h +41 -0
  53. data/include/prism/internal/allocator.h +68 -0
  54. data/include/prism/internal/allocator_debug.h +88 -0
  55. data/include/prism/internal/arena.h +108 -0
  56. data/include/prism/internal/bit.h +42 -0
  57. data/include/prism/internal/buffer.h +94 -0
  58. data/include/prism/internal/char.h +145 -0
  59. data/include/prism/internal/comments.h +54 -0
  60. data/include/prism/internal/constant_pool.h +152 -0
  61. data/include/prism/internal/diagnostic.h +389 -0
  62. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  63. data/include/prism/internal/integer.h +68 -0
  64. data/include/prism/internal/isinf.h +16 -0
  65. data/include/prism/internal/line_offset_list.h +34 -0
  66. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  67. data/include/prism/internal/magic_comments.h +23 -0
  68. data/include/prism/internal/memchr.h +15 -0
  69. data/include/prism/internal/node.h +42 -0
  70. data/include/prism/internal/options.h +232 -0
  71. data/include/prism/internal/parser.h +1015 -0
  72. data/include/prism/internal/regexp.h +26 -0
  73. data/include/prism/internal/serialize.h +39 -0
  74. data/include/prism/internal/source.h +72 -0
  75. data/include/prism/internal/static_literals.h +60 -0
  76. data/include/prism/internal/stringy.h +30 -0
  77. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  78. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  79. data/include/prism/internal/tokens.h +11 -0
  80. data/include/prism/internal/xxh3.h +16 -0
  81. data/include/prism/json.h +40 -0
  82. data/include/prism/line_offset_list.h +69 -0
  83. data/include/prism/magic_comments.h +53 -0
  84. data/include/prism/node.h +59 -49
  85. data/include/prism/node.hpp +272 -0
  86. data/include/prism/node_type.hpp +38 -0
  87. data/include/prism/options.h +154 -279
  88. data/include/prism/options.hpp +43 -0
  89. data/include/prism/parser.h +367 -870
  90. data/include/prism/parser.hpp +274 -0
  91. data/include/prism/prettyprint.h +14 -9
  92. data/include/prism/serialize.h +123 -0
  93. data/include/prism/source.h +163 -0
  94. data/include/prism/stream.h +36 -0
  95. data/include/prism/string_query.h +82 -0
  96. data/include/prism/stringy.h +61 -0
  97. data/include/prism/stringy.hpp +31 -0
  98. data/include/prism/version.h +20 -3
  99. data/include/prism.h +35 -320
  100. data/include/prism.hpp +24 -0
  101. data/lib/prism/compiler.rb +319 -305
  102. data/lib/prism/desugar_compiler.rb +105 -34
  103. data/lib/prism/dispatcher.rb +646 -473
  104. data/lib/prism/dot_visitor.rb +592 -511
  105. data/lib/prism/dsl.rb +391 -25
  106. data/lib/prism/ffi.rb +258 -90
  107. data/lib/prism/inspect_visitor.rb +357 -325
  108. data/lib/prism/lex_compat.rb +254 -212
  109. data/lib/prism/mutation_compiler.rb +313 -305
  110. data/lib/prism/node.rb +20966 -6992
  111. data/lib/prism/node_ext.rb +180 -187
  112. data/lib/prism/node_find.rb +215 -0
  113. data/lib/prism/parse_result/errors.rb +7 -1
  114. data/lib/prism/parse_result/newlines.rb +276 -8
  115. data/lib/prism/parse_result.rb +468 -139
  116. data/lib/prism/pattern.rb +65 -20
  117. data/lib/prism/reflection.rb +23 -12
  118. data/lib/prism/relocation.rb +149 -87
  119. data/lib/prism/serialize.rb +2889 -700
  120. data/lib/prism/string_query.rb +24 -1
  121. data/lib/prism/translation/parser/builder.rb +10 -2
  122. data/lib/prism/translation/parser/compiler.rb +44 -52
  123. data/lib/prism/translation/parser/lexer.rb +41 -92
  124. data/lib/prism/translation/parser.rb +1 -1
  125. data/lib/prism/translation/parser_current.rb +1 -1
  126. data/lib/prism/translation/ripper/lexer.rb +8 -10
  127. data/lib/prism/translation/ripper/sexp.rb +2 -10
  128. data/lib/prism/translation/ripper/shim.rb +2 -0
  129. data/lib/prism/translation/ripper.rb +967 -209
  130. data/lib/prism/translation/ruby_parser.rb +73 -316
  131. data/lib/prism/translation.rb +2 -0
  132. data/lib/prism/visitor.rb +632 -5
  133. data/lib/prism.rb +79 -9
  134. data/prism.gemspec +141 -62
  135. data/rbi/generated/prism/compiler.rbi +489 -0
  136. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  137. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  138. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  139. data/rbi/generated/prism/dsl.rbi +751 -0
  140. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  141. data/rbi/generated/prism/lex_compat.rbi +162 -0
  142. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  143. data/rbi/generated/prism/node.rbi +15143 -0
  144. data/rbi/generated/prism/node_ext.rbi +259 -0
  145. data/rbi/generated/prism/node_find.rbi +79 -0
  146. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  147. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  148. data/rbi/generated/prism/parse_result.rbi +776 -0
  149. data/rbi/generated/prism/pattern.rbi +134 -0
  150. data/rbi/generated/prism/reflection.rbi +89 -0
  151. data/rbi/generated/prism/relocation.rbi +329 -0
  152. data/rbi/generated/prism/serialize.rbi +185 -0
  153. data/rbi/generated/prism/string_query.rbi +43 -0
  154. data/rbi/generated/prism/translation.rbi +7 -0
  155. data/rbi/generated/prism/visitor.rbi +650 -0
  156. data/rbi/generated/prism.rbi +86 -0
  157. data/rbi/prism/translation/parser_versions.rbi +5 -0
  158. data/rbi/rubyvm/node_find.rbi +14 -0
  159. data/sig/generated/prism/compiler.rbs +492 -0
  160. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  161. data/sig/generated/prism/dispatcher.rbs +1292 -0
  162. data/sig/generated/prism/dot_visitor.rbs +635 -0
  163. data/sig/generated/prism/dsl.rbs +928 -0
  164. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  165. data/sig/generated/prism/lex_compat.rbs +166 -0
  166. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  167. data/sig/generated/prism/node.rbs +21632 -0
  168. data/sig/generated/prism/node_ext.rbs +290 -0
  169. data/sig/generated/prism/node_find.rbs +90 -0
  170. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  171. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  172. data/sig/generated/prism/parse_result.rbs +871 -0
  173. data/sig/generated/prism/pattern.rbs +156 -0
  174. data/sig/generated/prism/reflection.rbs +101 -0
  175. data/sig/generated/prism/relocation.rbs +428 -0
  176. data/sig/generated/prism/serialize.rbs +200 -0
  177. data/sig/generated/prism/string_query.rbs +43 -0
  178. data/sig/generated/prism/translation.rbs +8 -0
  179. data/sig/generated/prism/visitor.rbs +1111 -0
  180. data/sig/generated/prism.rbs +84 -0
  181. data/sig/manifest.yaml +2 -0
  182. data/src/arena.c +122 -0
  183. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  184. data/src/{util/pm_char.c → char.c} +28 -57
  185. data/src/comments.c +455 -0
  186. data/src/constant_pool.c +406 -0
  187. data/src/diagnostic.c +120 -68
  188. data/src/encoding.c +8 -3
  189. data/src/errors_format.c +519 -0
  190. data/src/{util/pm_integer.c → integer.c} +38 -22
  191. data/src/json.c +5733 -0
  192. data/src/line_offset_list.c +100 -0
  193. data/src/list.c +24 -0
  194. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  195. data/src/node.c +3465 -7787
  196. data/src/options.c +161 -54
  197. data/src/parser.c +371 -0
  198. data/src/prettyprint.c +421 -354
  199. data/src/prism.c +8503 -7280
  200. data/src/regexp.c +960 -45
  201. data/src/serialize.c +2434 -2048
  202. data/src/source.c +524 -0
  203. data/src/static_literals.c +303 -360
  204. data/src/string_query.c +180 -0
  205. data/src/stringy.c +61 -0
  206. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  207. data/src/strpbrk.c +447 -0
  208. data/src/{token_type.c → tokens.c} +39 -18
  209. data/src/xxh3.c +292 -0
  210. metadata +141 -63
  211. data/ext/prism/api_pack.c +0 -276
  212. data/include/prism/defines.h +0 -260
  213. data/include/prism/pack.h +0 -163
  214. data/include/prism/regexp.h +0 -43
  215. data/include/prism/static_literals.h +0 -121
  216. data/include/prism/util/pm_buffer.h +0 -236
  217. data/include/prism/util/pm_char.h +0 -204
  218. data/include/prism/util/pm_constant_pool.h +0 -218
  219. data/include/prism/util/pm_integer.h +0 -130
  220. data/include/prism/util/pm_memchr.h +0 -29
  221. data/include/prism/util/pm_newline_list.h +0 -113
  222. data/include/prism/util/pm_string.h +0 -200
  223. data/lib/prism/pack.rb +0 -230
  224. data/lib/prism/parse_result/comments.rb +0 -188
  225. data/rbi/prism/compiler.rbi +0 -12
  226. data/rbi/prism/dsl.rbi +0 -524
  227. data/rbi/prism/inspect_visitor.rbi +0 -12
  228. data/rbi/prism/node.rbi +0 -8750
  229. data/rbi/prism/node_ext.rbi +0 -107
  230. data/rbi/prism/parse_result.rbi +0 -404
  231. data/rbi/prism/reflection.rbi +0 -58
  232. data/rbi/prism/string_query.rbi +0 -12
  233. data/rbi/prism/visitor.rbi +0 -473
  234. data/rbi/prism.rbi +0 -63
  235. data/sig/prism/compiler.rbs +0 -9
  236. data/sig/prism/dispatcher.rbs +0 -19
  237. data/sig/prism/dot_visitor.rbs +0 -6
  238. data/sig/prism/dsl.rbs +0 -351
  239. data/sig/prism/inspect_visitor.rbs +0 -22
  240. data/sig/prism/lex_compat.rbs +0 -10
  241. data/sig/prism/node.rbs +0 -4042
  242. data/sig/prism/node_ext.rbs +0 -149
  243. data/sig/prism/pack.rbs +0 -43
  244. data/sig/prism/parse_result/comments.rbs +0 -38
  245. data/sig/prism/parse_result.rbs +0 -197
  246. data/sig/prism/pattern.rbs +0 -13
  247. data/sig/prism/reflection.rbs +0 -50
  248. data/sig/prism/relocation.rbs +0 -185
  249. data/sig/prism/serialize.rbs +0 -8
  250. data/sig/prism/string_query.rbs +0 -11
  251. data/sig/prism/visitor.rbs +0 -169
  252. data/sig/prism.rbs +0 -272
  253. data/src/pack.c +0 -509
  254. data/src/util/pm_constant_pool.c +0 -342
  255. data/src/util/pm_list.c +0 -49
  256. data/src/util/pm_newline_list.c +0 -125
  257. data/src/util/pm_string.c +0 -381
  258. data/src/util/pm_strpbrk.c +0 -206
data/docs/build_system.md CHANGED
@@ -87,7 +87,7 @@ If you need to use memory allocation functions implemented outside of the standa
87
87
  * Additionally, include `-I [path/to/custom_allocator]` where your `prism_xallocator.h` is located
88
88
  * Link the implementation of `prism_xallocator.c` that contains functions declared in `prism_xallocator.h`
89
89
 
90
- For further clarity, refer to `include/prism/defines.h`.
90
+ For further clarity, refer to `include/prism/internal/allocator.h`.
91
91
 
92
92
  ### Building prism from source as a C library
93
93
 
@@ -97,6 +97,7 @@ If you want to build prism as a shared library and link against it, you should c
97
97
 
98
98
  * `-fPIC -shared` - Compile as a shared library
99
99
  * `-DPRISM_EXPORT_SYMBOLS` - Export the symbols (by default nothing is exported)
100
+ * `-Wl,-Bsymbolic` (ELF platforms) - Bind the library's internal calls to its own exported functions at link time, so that they cannot be redirected to another copy of libprism loaded in the same process
100
101
 
101
102
  #### Flags
102
103
 
@@ -113,7 +114,6 @@ MAKEFLAGS="-j10" bundle exec rake compile
113
114
  * `PRISM_ENCODING_EXCLUDE_FULL` - Will cause the library to exclude the full encoding API, and only include the minimal number of encodings to support parsing Ruby code without encoding comments. By default this is off.
114
115
  * `PRISM_EXPORT_SYMBOLS` - Will cause the shared library to export symbols. By default this is off.
115
116
  * `PRISM_EXCLUDE_JSON` - Will cause the library to exclude the JSON API. By default this is off.
116
- * `PRISM_EXCLUDE_PACK` - Will cause the library to exclude the pack API. By default this is off.
117
117
  * `PRISM_EXCLUDE_PRETTYPRINT` - Will cause the library to exclude the prettyprint API. By default this is off.
118
118
  * `PRISM_EXCLUDE_SERIALIZATION` - Will cause the library to exclude the serialization API. By default this is off.
119
119
  * `PRISM_XALLOCATOR` - Will cause the library to use the custom memory allocator. By default this is off.
@@ -5,11 +5,12 @@ A lot of code in prism's repository is templated from a single configuration fil
5
5
  * `ext/prism/api_node.c` - for defining how to build Ruby objects for the nodes out of C structs
6
6
  * `include/prism/ast.h` - for defining the C structs that represent the nodes
7
7
  * `include/prism/diagnostic.h` - for defining the diagnostics
8
+ * `include/prism/node_new.h` - for defining the functions that create the nodes in C
8
9
  * `javascript/src/deserialize.js` - for defining how to deserialize the nodes in JavaScript
9
10
  * `javascript/src/nodes.js` - for defining the nodes in JavaScript
10
- * `java/org/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
11
- * `java/org/prism/Loader.java` - for defining how to deserialize the nodes in Java
12
- * `java/org/prism/Nodes.java` - for defining the nodes in Java
11
+ * `java/api/src/main/java-templates/org/ruby_lang/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
12
+ * `java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java` - for defining how to deserialize the nodes in Java
13
+ * `java/api/src/main/java-templates/org/ruby_lang/prism/Nodes.java` - for defining the nodes in Java
13
14
  * `lib/prism/compiler.rb` - for defining the compiler for the nodes in Ruby
14
15
  * `lib/prism/dispatcher.rb` - for defining the dispatch visitors for the nodes in Ruby
15
16
  * `lib/prism/dot_visitor.rb` - for defining the dot visitor for the nodes in Ruby
@@ -61,7 +62,7 @@ The available values for `type` are:
61
62
  * `constant` - A field that is an integer that represents an index in the constant pool. This is a `pm_constant_id_t` in C.
62
63
  * `constant[]` - A field that is an array of constants. This is a `pm_constant_id_list_t` in C.
63
64
  * `location` - A field that is a location. This is a `pm_location_t` in C.
64
- * `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `start` and `end` fields will be `NULL`.
65
+ * `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `length` field will be `0`.
65
66
  * `uint8` - A field that is an 8-bit unsigned integer. This is a `uint8_t` in C.
66
67
  * `uint32` - A field that is a 32-bit unsigned integer. This is a `uint32_t` in C.
67
68
 
data/docs/encoding.md CHANGED
@@ -107,7 +107,7 @@ For each of these encodings, prism provides functions for checking if the subseq
107
107
 
108
108
  ## Getting notified when the encoding changes
109
109
 
110
- You may want to get notified when the encoding changes based on the result of parsing an encoding comment. We use this internally for our `lex` function in order to provide the correct encodings for the tokens that are returned. For that you can register a callback with `pm_parser_register_encoding_changed_callback`. The callback will be called with a pointer to the parser. The encoding can be accessed through `parser->encoding`.
110
+ You may want to get notified when the encoding changes based on the result of parsing an encoding comment. We use this internally for our `lex` function in order to provide the correct encodings for the tokens that are returned. For that you can register a callback with `pm_parser_encoding_changed_callback_set`. The callback will be called with a pointer to the parser. The encoding can be accessed through `parser->encoding`.
111
111
 
112
112
  ```c
113
113
  // When the encoding that is being used to parse the source is changed by prism,
@@ -117,5 +117,5 @@ typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
117
117
  // Register a callback that will be called whenever prism changes the encoding
118
118
  // it is using to parse based on the magic comment.
119
119
  PRISM_EXPORTED_FUNCTION void
120
- pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
120
+ pm_parser_encoding_changed_callback_set(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
121
121
  ```
data/docs/fuzzing.md CHANGED
@@ -1,12 +1,11 @@
1
1
  # Fuzzing
2
2
 
3
- We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `fuzz` directory, which has the following structure:
3
+ We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `test/fuzz` directory, which has the following structure:
4
4
 
5
5
  ```
6
- fuzz
6
+ test/fuzz
7
7
  ├── corpus
8
- │   ├── parse fuzzing corpus for parsing (a symlink to our fixtures)
9
- │   └── regexp fuzzing corpus for regexp
8
+ │   └── parse fuzzing corpus for parsing (a symlink to our fixtures)
10
9
  ├── dict a AFL++ dictionary containing various tokens
11
10
  ├── docker
12
11
  │   └── Dockerfile for building a container with the fuzzer toolchain
@@ -14,8 +13,6 @@ fuzz
14
13
  ├── heisenbug.c entrypoint for reproducing a crash or hang
15
14
  ├── parse.c fuzz handler for parsing
16
15
  ├── parse.sh script to run parsing fuzzer
17
- ├── regexp.c fuzz handler for regular expression parsing
18
- ├── regexp.sh script to run regexp fuzzer
19
16
  └── tools
20
17
     ├── backtrace.sh generates backtrace files for a crash directory
21
18
     └── minimize.sh generates minimized crash or hang files
@@ -23,16 +20,14 @@ fuzz
23
20
 
24
21
  ## Usage
25
22
 
26
- There are currently three fuzzing targets
23
+ There is currently one fuzz target:
27
24
 
28
25
  - `pm_serialize_parse` (parse)
29
- - `pm_regexp_parse` (regexp)
30
26
 
31
- Respectively, fuzzing can be performed with
27
+ Fuzzing can be performed with
32
28
 
33
29
  ```
34
30
  make fuzz-run-parse
35
- make fuzz-run-regexp
36
31
  ```
37
32
 
38
33
  To end a fuzzing job, interrupt with CTRL+C. To enter a container with the fuzzing toolchain and debug utilities, run
@@ -43,8 +38,6 @@ make fuzz-debug
43
38
 
44
39
  # Out-of-bounds reads
45
40
 
46
- Currently, encoding functionality implementing the `pm_encoding_t` interface can read outside of inputs. For the time being, ASAN instrumentation is disabled for functions from src/enc. See `fuzz/asan.ignore`.
47
-
48
41
  To disable ASAN read instrumentation globally, use the `FUZZ_FLAGS` environment variable e.g.
49
42
 
50
43
  ```
@@ -55,7 +48,7 @@ Note, that this may make reproducing bugs difficult as they may depend on memory
55
48
 
56
49
  ```
57
50
  make fuzz-debug # enter the docker container with build tools
58
- make build/fuzz.heisenbug.parse # or .regexp
51
+ make build/fuzz.heisenbug.parse
59
52
  ./build/fuzz.heisenbug.parse path-to-problem-input
60
53
  ```
61
54
 
@@ -64,12 +57,12 @@ make build/fuzz.heisenbug.parse # or .regexp
64
57
  Triaging crashes and hangs is easier when the inputs are as short as possible. In the fuzz container, an entire crash or hang directory can be minimized using
65
58
 
66
59
  ```
67
- ./fuzz/tools/minimize.sh <directory>
60
+ ./test/fuzz/tools/minimize.sh <directory>
68
61
  ```
69
62
 
70
63
  e.g.
71
64
  ```
72
- ./fuzz/tools/minimize.sh fuzz/output/parse/default/crashes
65
+ ./test/fuzz/tools/minimize.sh test/fuzz/output/parse/default/crashes
73
66
  ```
74
67
 
75
68
  This may take a long time. In the crash/hang directory, for each input file there will appear a minimized version with the extension `.min` appended.
@@ -77,7 +70,7 @@ This may take a long time. In the crash/hang directory, for each input file ther
77
70
  Backtraces for crashes (not hangs) can be generated en masse with
78
71
 
79
72
  ```
80
- ./fuzz/tools/backtrace.sh <directory>
73
+ ./test/fuzz/tools/backtrace.sh <directory>
81
74
  ```
82
75
 
83
76
  Files with basename equal to the input file name with extension `.bt` will be created e.g.
data/docs/javascript.md CHANGED
@@ -16,10 +16,25 @@ Then import the package:
16
16
  import { loadPrism } from "@ruby/prism";
17
17
  ```
18
18
 
19
- Then call the load function to get a parse function:
19
+ Then call the load function to get a parse function. It takes the source as
20
+ bytes, in the encoding the source is written in, because that is what the parser
21
+ reads and what the locations in the result are offsets into. If you are starting
22
+ from a string, encode it first:
20
23
 
21
24
  ```js
22
- const parse = await loadPrism();
25
+ const parseArray = await loadPrism();
26
+
27
+ function parse(source) {
28
+ return parseArray(new TextEncoder().encode(source));
29
+ }
30
+ ```
31
+
32
+ Reading a file gives you those bytes already, so there is nothing to encode:
33
+
34
+ ```js
35
+ import { readFile } from "node:fs/promises";
36
+
37
+ const parseResult = parseArray(await readFile("example.rb"));
23
38
  ```
24
39
 
25
40
  ## Browser
@@ -56,7 +71,7 @@ Finally, you can create a function that will parse a string of Ruby code.
56
71
 
57
72
  ```js
58
73
  function parse(source) {
59
- return parsePrism(instance.exports, source);
74
+ return parsePrism(instance.exports, new TextEncoder().encode(source));
60
75
  }
61
76
  ```
62
77
 
@@ -84,7 +99,7 @@ Here's an example of a custom `FooCalls` visitor:
84
99
  import { loadPrism, Visitor } from "@ruby/prism"
85
100
 
86
101
  const parse = await loadPrism();
87
- const parseResult = parse("foo()");
102
+ const parseResult = parse(new TextEncoder().encode("foo()"));
88
103
 
89
104
  class FooCalls extends Visitor {
90
105
  visitCallNode(node) {
data/docs/mapping.md CHANGED
@@ -112,6 +112,6 @@ The following table shows how the various CRuby nodes are mapped to prism nodes.
112
112
  | `NODE_ARYPTN` | `PM_ARRAY_PATTERN_NODE` |
113
113
  | `NODE_HSHPTN` | `PM_HASH_PATTERN_NODE` |
114
114
  | `NODE_FNDPTN` | `PM_FIND_PATTERN_NODE` |
115
- | `NODE_ERROR` | `PM_MISSING_NODE` |
115
+ | `NODE_ERROR` | `PM_ERROR_RECOVERY_NODE` |
116
116
  | `NODE_LAST` | |
117
117
  ```
data/docs/releasing.md CHANGED
@@ -23,9 +23,10 @@ ruby -pi -e 'gsub(/PRISM_VERSION_PATCH \d+/, %Q{PRISM_VERSION_PATCH #{ENV["PRISM
23
23
  ruby -pi -e 'gsub(/PRISM_VERSION ".+?"/, %Q{PRISM_VERSION "#{ENV["PRISM_VERSION"]}"})' include/prism/version.h
24
24
  ruby -pi -e 'gsub(/"version": ".+?"/, %Q{"version": "#{ENV["PRISM_VERSION"]}"})' javascript/package.json
25
25
  ruby -pi -e 'gsub(/lossy\(\), ".+?"/, %Q{lossy(), "#{ENV["PRISM_VERSION"]}"})' rust/ruby-prism-sys/tests/utils_tests.rs
26
- ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/org/prism/Loader.java.erb
27
- ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/org/prism/Loader.java.erb
28
- ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/org/prism/Loader.java.erb
26
+ ruby -pi -e 'gsub(/assertEquals\(".+?", prism.version\(\)\)/, %Q{assertEquals("#{ENV["PRISM_VERSION"]}", prism.version())})' java/wasm/src/test/java/org/jruby/parser/prism/WASMTest.java
27
+ ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java.erb
28
+ ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java.erb
29
+ ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java.erb
29
30
  ruby -pi -e 'gsub(/MAJOR_VERSION = \d+/, %Q{MAJOR_VERSION = #{ENV["PRISM_MAJOR"]}})' templates/javascript/src/deserialize.js.erb
30
31
  ruby -pi -e 'gsub(/MINOR_VERSION = \d+/, %Q{MINOR_VERSION = #{ENV["PRISM_MINOR"]}})' templates/javascript/src/deserialize.js.erb
31
32
  ruby -pi -e 'gsub(/PATCH_VERSION = \d+/, %Q{PATCH_VERSION = #{ENV["PRISM_PATCH"]}})' templates/javascript/src/deserialize.js.erb
@@ -57,6 +58,8 @@ chruby ruby-4.1.0-dev && BUNDLE_GEMFILE=gemfiles/4.1/Gemfile bundle install
57
58
  bundle exec rake cargo:build
58
59
  ```
59
60
 
61
+ * Regenerate `rust/ruby-prism-sys/src/bindings.rs` with `bundle exec rake cargo:bindings`. CI fails if the committed file is stale.
62
+
60
63
  * Commit all of the updated files:
61
64
 
62
65
  ```sh
@@ -71,5 +74,74 @@ git push
71
74
 
72
75
  ## Publishing
73
76
 
74
- * Update the GitHub release page with a copy of the latest entry in the `CHANGELOG.md` file.
75
- * Push a new tag to the GitHub repository, following the `vX.Y.Z` format.
77
+ ### Automatic (Github Actions)
78
+
79
+ The [`publish-gem.yml`](../.github/workflows/publish-gem.yml) workflow handles publishing. It triggers either on a `v*` tag push or via manual `workflow_dispatch`. In both cases it:
80
+
81
+ - Builds the source gem and all native gems (calling the reusable [`build-gems.yml`](../.github/workflows/build-gems.yml) workflow with `--release`).
82
+ - Pushes every gem to RubyGems.org using OIDC trusted publishing (no API key required).
83
+ - Creates (or updates) a GitHub release for the tag, attaches the `.gem` files and a `CHECKSUMS.txt`, and auto-generates release notes.
84
+
85
+ The `push` job is gated on `github.repository == 'ruby/prism'` and runs in the `rubygems.org` GitHub environment, which is the trust anchor for OIDC publishing to RubyGems.org. Pushes that already exist on RubyGems.org are reported as warnings rather than failing the run, so re-runs after a partial failure are safe.
86
+
87
+ #### 1. Trigger the release pipeline
88
+
89
+ After the preparation commit is on `main`, tag and push:
90
+
91
+ ```sh
92
+ git tag "v${PRISM_VERSION}"
93
+ git push origin "v${PRISM_VERSION}"
94
+ ```
95
+
96
+ The `v*` tag push triggers `publish-gem.yml` automatically.
97
+
98
+ If you need to re-trigger the workflow against an existing tag — for example, if the tag-push run failed partway through and you've fixed the cause — go to the Actions tab and use "Run workflow" on `Publish gem to rubygems.org`, passing the tag (e.g. `v1.10.0`) as the `version_tag` input. The workflow will check out that tag, rebuild, and push.
99
+
100
+ #### 2. Edit the release notes
101
+
102
+ The workflow creates the GitHub release with auto-generated notes. Edit it to paste in the relevant section of `CHANGELOG.md`.
103
+
104
+ ### Manually
105
+
106
+ If for whatever reason (like an emergency), you need to release a version and the Github Actions pipelines aren't available, here's how you can build the full set of native gems.
107
+
108
+ #### 1. Build gems
109
+
110
+ ```sh
111
+ bin/build-gems
112
+ ```
113
+
114
+ This script will:
115
+ - Run `bundle update` and `bundle package` to vendor dependencies
116
+ - Run a safety check (`compile` and `test`)
117
+ - Build all gems via `rake gem:all` (source gem + native gems for all platforms using rake-compiler-dock)
118
+ - Place all gems in `./gems/`
119
+ - Verify all built gems with `test/prism/packaging/test-gem-file-contents`
120
+ - Print SHA256 checksums for inclusion in release notes
121
+
122
+ #### 2. Push gems
123
+
124
+ Push the native gems first, then the source gem last (so that when users see the new version, native gems are already available):
125
+
126
+ ```sh
127
+ # push native gems first
128
+ for gem in gems/prism-*-*.gem ; do
129
+ gem push "$gem"
130
+ done
131
+
132
+ # push source gem last
133
+ gem push gems/prism-${PRISM_VERSION}.gem
134
+ ```
135
+
136
+ #### 3. Push tag
137
+
138
+ Push a new tag to the GitHub repository, following the `vX.Y.Z` format:
139
+
140
+ ```sh
141
+ git tag "v${PRISM_VERSION}"
142
+ git push origin "v${PRISM_VERSION}"
143
+ ```
144
+
145
+ #### 4. Create a release
146
+
147
+ Update the GitHub release page with a copy of the latest entry in the `CHANGELOG.md` file, including the SHA256 checksums from the build output.
data/docs/relocation.md CHANGED
@@ -6,10 +6,10 @@ The `Relocation` module provides an API around this property. It allows you to "
6
6
 
7
7
  ## Getting started
8
8
 
9
- To get started with the `Relocation` module, you would first instantiate a `Repository` object. You do this through a DSL that chains method calls for configuration. For example, if for every entry in the repository you want to store the start and end lines, the start and end code unit columns for in UTF-16, and the leading comments, you would:
9
+ To get started with the `Relocation` module, you would first instantiate a `Repository` object. You do this through a DSL that chains method calls for configuration. For example, if for every entry in the repository you want to store the start and end lines, and the start and end code unit columns for in UTF-16, you would:
10
10
 
11
11
  ```ruby
12
- repository = Prism::Relocation.filepath("path/to/file").lines.code_unit_columns(Encoding::UTF_16).leading_comments
12
+ repository = Prism::Relocation.filepath("path/to/file").lines.code_unit_columns(Encoding::UTF_16)
13
13
  ```
14
14
 
15
15
  Now that you have the repository, you can pass it into any of the `save*` APIs on nodes or locations to create entries in the repository that will be lazily reified.
@@ -27,8 +27,6 @@ entry.end_line
27
27
 
28
28
  entry.start_code_units_column
29
29
  entry.end_code_units_column
30
-
31
- entry.leading_comments
32
30
  ```
33
31
 
34
32
  Note that if you had configured other fields to be saved, you would be able to access them as well. The first time one of these fields is accessed, the repository will reify every entry it knows about and then clean itself up. In this way, you can effectively treat them as if you had kept around lightweight versions of `Prism::Node` or `Prism::Location` objects.
@@ -44,6 +44,17 @@ The comment type is one of:
44
44
  | `1` | comment type |
45
45
  | location | the location in the source of this comment |
46
46
 
47
+ ### comment attachment
48
+
49
+ A comment attachment lists the comments owned by a single node.
50
+ Only nodes that own comments have an attachment, and there are none unless the `attach_comments` option was set.
51
+
52
+ | # bytes | field |
53
+ | --- | --- |
54
+ | varuint | the node identifier of the node that owns the comments |
55
+ | varuint | number of comments owned by the node |
56
+ | varuint* | the index of each owned comment in the list of comments, ordered by start offset |
57
+
47
58
  ### magic comment
48
59
 
49
60
  | # bytes | field |
@@ -95,10 +106,13 @@ The header is structured like the following table:
95
106
  | `1` | 1 indicates only semantics fields were serialized, 0 indicates all fields were serialized (including location fields) |
96
107
  | string | the encoding name |
97
108
  | varsint | the start line |
109
+ | `8` | the checksum of the source up to the end of the `__END__` line in native endian order, see `pm_parser_checksum` |
98
110
  | varuint | number of newline offsets |
99
111
  | varuint* | newline offsets |
100
112
  | varuint | number of comments |
101
113
  | comment* | comments |
114
+ | varuint | number of nodes with attached comments |
115
+ | comment attachment* | comment attachments |
102
116
  | varuint | number of magic comments |
103
117
  | magic comment* | magic comments |
104
118
  | location? | the optional location of the `__END__` keyword and its contents |
@@ -106,12 +120,19 @@ The header is structured like the following table:
106
120
  | error* | errors |
107
121
  | varuint | number of warnings |
108
122
  | warning* | warnings |
123
+ | `1` | `1` if the source is continuable (incomplete but could become valid with more input), `0` otherwise |
109
124
  | `4` | content pool offset |
110
125
  | varuint | content pool size |
111
126
 
112
127
  After the header comes the body of the serialized string.
113
- The body consists of a sequence of nodes that is built using a prefix traversal order of the syntax tree.
114
- Each node is structured like the following table:
128
+ The body consists of a sequence of nodes that is built using a postfix traversal order of the syntax tree: the children of each node come before the node itself, so the root node comes last.
129
+ The body ends at the content pool offset specified in the header.
130
+
131
+ The children of a node are the values of its `node`, `node?`, and `node[]` fields, which are serialized in the order in which the fields are listed in `config.yml`.
132
+ A `node` field is serialized as its node, a `node?` field is serialized as its node or as a single `0` byte if the node is not present, and a `node[]` field is serialized as each of its nodes in order.
133
+ A loader can therefore keep a stack of loaded nodes: a `0` byte pushes an absent node, and each node pops its children off the top of the stack (they are in field order) and then pushes itself.
134
+
135
+ After its children, each node is structured like the following table:
115
136
 
116
137
  | # bytes | field |
117
138
  | --- | --- |
@@ -120,13 +141,12 @@ Each node is structured like the following table:
120
141
  | location | node location |
121
142
  | varuint | node flags |
122
143
 
123
- Every field on the node is then appended to the serialized string. The fields can be determined by referencing `config.yml`. Depending on the type of field, it could take a couple of different forms, described below:
144
+ Every other field on the node is then appended to the serialized string. The fields can be determined by referencing `config.yml`. Depending on the type of field, it could take a couple of different forms, described below:
124
145
 
125
146
  * `double` - A field that is a `double`. This is structured as a sequence of 8 bytes in native endian order.
126
- * `node` - A field that is a node. This is structured just as like parent node.
127
- * `node?` - A field that is a node that is optionally present. If the node is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just as like parent node.
128
- * `node[]` - A field that is an array of nodes. This is structured as a variable-length integer length, followed by the child nodes themselves.
129
- * `string` - A field that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is structured as a variable-length integer byte length, followed by the string itself (_without_ a trailing null byte).
147
+ * `node`, `node?` - Nothing is appended here, since the node has already been serialized as a child.
148
+ * `node[]` - A field that is an array of nodes. This is structured as a variable-length integer length, which is the number of children that belong to this field.
149
+ * `string` - A field that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is structured as a variable-length integer byte length, followed by the string bytes (_without_ a trailing null byte).
130
150
  * `constant` - A variable-length integer that represents an index in the constant pool.
131
151
  * `constant?` - An optional variable-length integer that represents an index in the constant pool. If it's not present, then a single `0` byte will be written in its place.
132
152
  * `integer` - A field that represents an arbitrary-sized integer. The structure is listed above.
@@ -135,23 +155,22 @@ Every field on the node is then appended to the serialized string. The fields ca
135
155
  * `uint8` - A field that is an 8-bit unsigned integer. This is structured as a single byte.
136
156
  * `uint32` - A field that is a 32-bit unsigned integer. This is structured as a variable-length integer.
137
157
 
138
- After the syntax tree, the content pool is serialized. This is a list of constants that were referenced from within the tree. The content pool begins at the offset specified in the header. Constants can be either "owned" (in which case their contents are embedded in the serialization) or "shared" (in which case their contents represent a slice of the source string). The most significant bit of the constant indicates whether it is owned or shared.
139
-
140
- In the case that it is owned, the constant is structured as follows:
158
+ The `body` field of a `DefNode` is lazy, which means that loaders can skip it and load it later. The children of a `DefNode` are preceded by the following, which loaders that do not skip lazy fields can ignore:
141
159
 
142
160
  | # bytes | field |
143
161
  | --- | --- |
144
- | `4` | the byte offset in the serialization for the contents of the constant |
145
- | `4` | the byte length in the serialization |
162
+ | `1` | `255` |
163
+ | `4` | the byte length of the children that come before the `body` child |
164
+ | `4` | the byte length of the `body` child |
146
165
 
147
- Note that you will need to mask off the most significant bit for the byte offset in the serialization. In the case that it is shared, the constant is structured as follows:
166
+ After the syntax tree, the content pool is serialized. This is a list of constants that were referenced from within the tree. The content pool begins at the offset specified in the header. Every constant is embedded in the serialization. Each constant is structured as follows:
148
167
 
149
168
  | # bytes | field |
150
169
  | --- | --- |
151
- | `4` | the byte offset in the source string for the contents of the constant |
152
- | `4` | the byte length in the source string |
170
+ | `4` | the byte offset in the serialization for the contents of the constant |
171
+ | `4` | the byte length in the serialization |
153
172
 
154
- After the constant pool, the contents of the owned constants are serialized. This is just a sequence of bytes that represent the contents of the constants. At the end of the serialization, the buffer is null terminated.
173
+ After the constant pool, the contents of the constants are serialized. This is just a sequence of bytes that represent the contents of the constants. At the end of the serialization, the buffer is null terminated.
155
174
 
156
175
  ## APIs
157
176
 
@@ -161,31 +180,23 @@ The relevant APIs and struct definitions are listed below:
161
180
  // A pm_buffer_t is a simple memory buffer that stores data in a contiguous
162
181
  // block of memory. It is used to store the serialized representation of a
163
182
  // prism tree.
164
- typedef struct {
165
- char *value;
166
- size_t length;
167
- size_t capacity;
168
- } pm_buffer_t;
169
-
170
- // Free the memory associated with the buffer.
171
- void pm_buffer_free(pm_buffer_t *);
172
183
 
173
184
  // Parse and serialize the AST represented by the given source to the given
174
185
  // buffer.
175
186
  void pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t length, const char *data);
176
187
  ```
177
188
 
178
- Typically you would use a stack-allocated `pm_buffer_t` and call `pm_serialize_parse`, as in:
189
+ Typically you would allocate a `pm_buffer_t` and call `pm_serialize_parse`, as in:
179
190
 
180
191
  ```c
181
192
  void
182
193
  serialize(const uint8_t *source, size_t length) {
183
- pm_buffer_t buffer = { 0 };
184
- pm_serialize_parse(&buffer, source, length, NULL);
194
+ pm_buffer_t *buffer = pm_buffer_new();
195
+ pm_serialize_parse(buffer, source, length, NULL);
185
196
 
186
197
  // Do something with the serialized string.
187
198
 
188
- pm_buffer_free(&buffer);
199
+ pm_buffer_free(buffer);
189
200
  }
190
201
  ```
191
202
 
data/docs/testing.md CHANGED
@@ -14,42 +14,59 @@ These test specific prism implementation details like comments, errors, and regu
14
14
 
15
15
  Snapshot tests ensure that parsed output is equivalent to previous parsed output. There are many categorized examples of valid syntax within the `test/prism/fixtures/` directory. When the test suite runs, it will parse all of this syntax, and compare it against corresponding files in the `test/prism/snapshots/` directory. For example, `test/prism/fixtures/strings.txt` has a corresponding `test/prism/snapshots/strings.txt`.
16
16
 
17
- If the parsed files do not match, it will raise an error. If there is not a corresponding file in the `test/prism/snapshots/` directory, one will be created so that it exists for the next test run.
17
+ If the parsed files do not match, it will raise an error. If there is not a corresponding file in the `test/prism/snapshots/` directory, one will be created so that it exists for the next test run. If the snapshot already exists but differs you can run with `UPDATE_SNAPSHOTS=1` to regenerate them.
18
18
 
19
- ### Testing against repositories
19
+ When working on a specific example, you can limit tests to a single fixture via `FOCUS=strings.txt`.
20
20
 
21
- To test the parser against a repository, you can run `FILEPATHS='/path/to/repository/**/*.rb' rake lex`. This will run the parser against every file matched by the glob pattern and check its generated tokens against those generated by ripper.
21
+ #### Versioning
22
22
 
23
- ## Local testing
24
-
25
- As you are working, you will likely want to test your code locally. `test.rb` is ignored by git, so it can be used for local testing. There are also two executables which may help you:
26
-
27
- 1. **bin/lex** takes a filepath and compares prism's lexed output to Ripper's lexed output. It prints any lexed output that doesn't match. It does some minor transformations to the lexed output in order to compare them, like split prism's heredoc tokens to mirror Ripper's.
28
-
29
- ```
30
- $ bin/lex test.rb
31
- ```
23
+ Prism is capable of parsing code with different version specifiers. The test suite reflects this:
24
+ * Test files under `test/prism/snapshots/*.txt` are validated against all supported versions.
25
+ * Test files under `test/prism/snapshots/[MAJOR].[MINOR]/*.txt` are validated against all versions starting from the given version.
26
+ * Test files under `test/prism/snapshots/[MAJOR].[MINOR]-[MAJOR].[MINOR]/*.txt` are validated against the given version range.
32
27
 
33
- If you would like to see the full lexed comparison, and not only the output that doesn't match, you can run with `VERBOSE=1`:
34
-
35
- ```
36
- $ VERBOSE=1 bin/lex test.rb
37
- ```
28
+ Should you add logic specific to a new ruby version, validate the new behavior but also that old behavior remains the same.
38
29
 
39
- `bin/lex` can also be used with `-e` and then source code, like this:
30
+ ### Error tests
40
31
 
41
- ```
42
- $ bin/lex -e "1 + 2"
43
- ```
32
+ Error tests are similar to snapshot tests and expectations are located under `test/prism/errors/`.
33
+ They contain invalid syntax and rendered errors, similar to how `ruby -c` would show them.
34
+ Error tests are versioned in the same way that snapshot tests are and also support `FOCUS`/`UPDATE_SNAPSHOTS`.
44
35
 
45
- 2. **bin/parse** takes a filepath and outputs prism's parsed node structure generated from reading the file.
36
+ ## Local testing
46
37
 
38
+ As you are working, you will likely want to test your code locally. `test.rb` is ignored by git, so it can be used for local testing. **bin/prism** contains a multitude of options to aid you, depending on which area
39
+ of prism you are working on. `bin/prism --help` prints a list of all available commands.
40
+
41
+ Commands that take a `[source]` can be called in three different ways:
42
+ 1. `bin/prism parse` will use `test.rb` as input.
43
+ 1. `bin/prism parse foo.rb` will read from the given path.
44
+ 1. `bin/prism parse -e "1 + 2"` will use the given source code.
45
+
46
+ `bin/prism parse` is aliased as `bin/parse` for for convenience.
47
+
48
+ ### Validating your changes against real code
49
+
50
+ If you create a change which is not expected to have a large impact on real code, you can use
51
+ `bin/compare` to validate that assumption. It compiles two versions of prism (for example
52
+ your feature branch and main), comparing ast, errors and warnings. Once done, a report is
53
+ printed showcasing in what way files changed, if any (syntax valid -> syntax invalid, changed AST, etc.)
54
+
55
+ ```sh
56
+ $ bin/compare main feature-branch ~/all-of-rubygems`
57
+ ...
58
+ 0/3157385
59
+ 1000/3157385
60
+ ...
61
+ 3157000/3157385
62
+ Oops:
63
+ intermine-1.05.00/lib/intermine/lists.rb changed from valid(true) to valid(false)
64
+ whistle-0.1.1/lib/resource.rb changed from valid(true) to valid(false)
65
+ Took 2661.5390081949999512 seconds
47
66
  ```
48
- $ bin/parse test.rb
49
- ```
50
-
51
- `bin/parse` can also be used with `-e` and then source code, like this:
52
67
 
53
- ```
54
- $ bin/parse -e "1 + 2"
55
- ```
68
+ It is up to you to aquire code samples. Here are some resources you can use for this purpose:
69
+ * https://github.com/jeromedalbert/real-world-ruby-apps
70
+ * https://github.com/eliotsykes/real-world-rails
71
+ * https://github.com/rubygems/rubygems-mirror
72
+ Requires some work to unpack the downloaded gems. You should also run with `RUBYGEMS_MIRROR_ONLY_LATEST=TRUE`.