prism 1.9.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -40
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +49 -20
  6. data/README.md +38 -98
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +9 -16
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +39 -28
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1527 -1467
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +759 -418
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +45 -0
  23. data/include/prism/arena.hpp +42 -0
  24. data/include/prism/ast.h +2738 -611
  25. data/include/prism/ast.hpp +641 -0
  26. data/include/prism/buffer.h +60 -0
  27. data/include/prism/buffer.hpp +55 -0
  28. data/include/prism/comments.h +72 -0
  29. data/include/prism/compiler/accel.h +26 -0
  30. data/include/prism/compiler/align.h +36 -0
  31. data/include/prism/compiler/assume.h +25 -0
  32. data/include/prism/compiler/branch.h +26 -0
  33. data/include/prism/compiler/bswap.h +46 -0
  34. data/include/prism/compiler/cplusplus.hpp +16 -0
  35. data/include/prism/compiler/endianness.h +34 -0
  36. data/include/prism/compiler/exceptions.hpp +17 -0
  37. data/include/prism/compiler/exported.h +24 -0
  38. data/include/prism/compiler/fallthrough.h +22 -0
  39. data/include/prism/compiler/filesystem.h +32 -0
  40. data/include/prism/compiler/flex_array.h +19 -0
  41. data/include/prism/compiler/force_inline.h +21 -0
  42. data/include/prism/compiler/format.h +27 -0
  43. data/include/prism/compiler/inline.h +17 -0
  44. data/include/prism/compiler/lifetimebound.hpp +25 -0
  45. data/include/prism/compiler/nodiscard.h +22 -0
  46. data/include/prism/compiler/nonnull.h +18 -0
  47. data/include/prism/compiler/unused.h +18 -0
  48. data/include/prism/constant_pool.h +106 -0
  49. data/include/prism/diagnostic.h +55 -402
  50. data/include/prism/errors_format.h +61 -0
  51. data/include/prism/excludes.h +29 -0
  52. data/include/prism/integer.h +41 -0
  53. data/include/prism/internal/allocator.h +68 -0
  54. data/include/prism/internal/allocator_debug.h +88 -0
  55. data/include/prism/internal/arena.h +108 -0
  56. data/include/prism/internal/bit.h +42 -0
  57. data/include/prism/internal/buffer.h +94 -0
  58. data/include/prism/internal/char.h +145 -0
  59. data/include/prism/internal/comments.h +54 -0
  60. data/include/prism/internal/constant_pool.h +152 -0
  61. data/include/prism/internal/diagnostic.h +389 -0
  62. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  63. data/include/prism/internal/integer.h +68 -0
  64. data/include/prism/internal/isinf.h +16 -0
  65. data/include/prism/internal/line_offset_list.h +34 -0
  66. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  67. data/include/prism/internal/magic_comments.h +23 -0
  68. data/include/prism/internal/memchr.h +15 -0
  69. data/include/prism/internal/node.h +42 -0
  70. data/include/prism/internal/options.h +232 -0
  71. data/include/prism/internal/parser.h +1015 -0
  72. data/include/prism/internal/regexp.h +26 -0
  73. data/include/prism/internal/serialize.h +39 -0
  74. data/include/prism/internal/source.h +72 -0
  75. data/include/prism/internal/static_literals.h +60 -0
  76. data/include/prism/internal/stringy.h +30 -0
  77. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  78. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  79. data/include/prism/internal/tokens.h +11 -0
  80. data/include/prism/internal/xxh3.h +16 -0
  81. data/include/prism/json.h +40 -0
  82. data/include/prism/line_offset_list.h +69 -0
  83. data/include/prism/magic_comments.h +53 -0
  84. data/include/prism/node.h +59 -49
  85. data/include/prism/node.hpp +272 -0
  86. data/include/prism/node_type.hpp +38 -0
  87. data/include/prism/options.h +154 -279
  88. data/include/prism/options.hpp +43 -0
  89. data/include/prism/parser.h +367 -870
  90. data/include/prism/parser.hpp +274 -0
  91. data/include/prism/prettyprint.h +14 -9
  92. data/include/prism/serialize.h +123 -0
  93. data/include/prism/source.h +163 -0
  94. data/include/prism/stream.h +36 -0
  95. data/include/prism/string_query.h +82 -0
  96. data/include/prism/stringy.h +61 -0
  97. data/include/prism/stringy.hpp +31 -0
  98. data/include/prism/version.h +20 -3
  99. data/include/prism.h +35 -320
  100. data/include/prism.hpp +24 -0
  101. data/lib/prism/compiler.rb +319 -305
  102. data/lib/prism/desugar_compiler.rb +105 -34
  103. data/lib/prism/dispatcher.rb +646 -473
  104. data/lib/prism/dot_visitor.rb +592 -511
  105. data/lib/prism/dsl.rb +391 -25
  106. data/lib/prism/ffi.rb +258 -90
  107. data/lib/prism/inspect_visitor.rb +357 -325
  108. data/lib/prism/lex_compat.rb +254 -212
  109. data/lib/prism/mutation_compiler.rb +313 -305
  110. data/lib/prism/node.rb +20966 -6992
  111. data/lib/prism/node_ext.rb +180 -187
  112. data/lib/prism/node_find.rb +215 -0
  113. data/lib/prism/parse_result/errors.rb +7 -1
  114. data/lib/prism/parse_result/newlines.rb +276 -8
  115. data/lib/prism/parse_result.rb +468 -139
  116. data/lib/prism/pattern.rb +65 -20
  117. data/lib/prism/reflection.rb +23 -12
  118. data/lib/prism/relocation.rb +149 -87
  119. data/lib/prism/serialize.rb +2889 -700
  120. data/lib/prism/string_query.rb +24 -1
  121. data/lib/prism/translation/parser/builder.rb +10 -2
  122. data/lib/prism/translation/parser/compiler.rb +44 -52
  123. data/lib/prism/translation/parser/lexer.rb +41 -92
  124. data/lib/prism/translation/parser.rb +1 -1
  125. data/lib/prism/translation/parser_current.rb +1 -1
  126. data/lib/prism/translation/ripper/lexer.rb +8 -10
  127. data/lib/prism/translation/ripper/sexp.rb +2 -10
  128. data/lib/prism/translation/ripper/shim.rb +2 -0
  129. data/lib/prism/translation/ripper.rb +967 -209
  130. data/lib/prism/translation/ruby_parser.rb +73 -316
  131. data/lib/prism/translation.rb +2 -0
  132. data/lib/prism/visitor.rb +632 -5
  133. data/lib/prism.rb +79 -9
  134. data/prism.gemspec +141 -62
  135. data/rbi/generated/prism/compiler.rbi +489 -0
  136. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  137. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  138. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  139. data/rbi/generated/prism/dsl.rbi +751 -0
  140. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  141. data/rbi/generated/prism/lex_compat.rbi +162 -0
  142. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  143. data/rbi/generated/prism/node.rbi +15143 -0
  144. data/rbi/generated/prism/node_ext.rbi +259 -0
  145. data/rbi/generated/prism/node_find.rbi +79 -0
  146. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  147. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  148. data/rbi/generated/prism/parse_result.rbi +776 -0
  149. data/rbi/generated/prism/pattern.rbi +134 -0
  150. data/rbi/generated/prism/reflection.rbi +89 -0
  151. data/rbi/generated/prism/relocation.rbi +329 -0
  152. data/rbi/generated/prism/serialize.rbi +185 -0
  153. data/rbi/generated/prism/string_query.rbi +43 -0
  154. data/rbi/generated/prism/translation.rbi +7 -0
  155. data/rbi/generated/prism/visitor.rbi +650 -0
  156. data/rbi/generated/prism.rbi +86 -0
  157. data/rbi/prism/translation/parser_versions.rbi +5 -0
  158. data/rbi/rubyvm/node_find.rbi +14 -0
  159. data/sig/generated/prism/compiler.rbs +492 -0
  160. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  161. data/sig/generated/prism/dispatcher.rbs +1292 -0
  162. data/sig/generated/prism/dot_visitor.rbs +635 -0
  163. data/sig/generated/prism/dsl.rbs +928 -0
  164. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  165. data/sig/generated/prism/lex_compat.rbs +166 -0
  166. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  167. data/sig/generated/prism/node.rbs +21632 -0
  168. data/sig/generated/prism/node_ext.rbs +290 -0
  169. data/sig/generated/prism/node_find.rbs +90 -0
  170. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  171. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  172. data/sig/generated/prism/parse_result.rbs +871 -0
  173. data/sig/generated/prism/pattern.rbs +156 -0
  174. data/sig/generated/prism/reflection.rbs +101 -0
  175. data/sig/generated/prism/relocation.rbs +428 -0
  176. data/sig/generated/prism/serialize.rbs +200 -0
  177. data/sig/generated/prism/string_query.rbs +43 -0
  178. data/sig/generated/prism/translation.rbs +8 -0
  179. data/sig/generated/prism/visitor.rbs +1111 -0
  180. data/sig/generated/prism.rbs +84 -0
  181. data/sig/manifest.yaml +2 -0
  182. data/src/arena.c +122 -0
  183. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  184. data/src/{util/pm_char.c → char.c} +28 -57
  185. data/src/comments.c +455 -0
  186. data/src/constant_pool.c +406 -0
  187. data/src/diagnostic.c +120 -68
  188. data/src/encoding.c +8 -3
  189. data/src/errors_format.c +519 -0
  190. data/src/{util/pm_integer.c → integer.c} +38 -22
  191. data/src/json.c +5733 -0
  192. data/src/line_offset_list.c +100 -0
  193. data/src/list.c +24 -0
  194. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  195. data/src/node.c +3465 -7787
  196. data/src/options.c +161 -54
  197. data/src/parser.c +371 -0
  198. data/src/prettyprint.c +421 -354
  199. data/src/prism.c +8503 -7280
  200. data/src/regexp.c +960 -45
  201. data/src/serialize.c +2434 -2048
  202. data/src/source.c +524 -0
  203. data/src/static_literals.c +303 -360
  204. data/src/string_query.c +180 -0
  205. data/src/stringy.c +61 -0
  206. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  207. data/src/strpbrk.c +447 -0
  208. data/src/{token_type.c → tokens.c} +39 -18
  209. data/src/xxh3.c +292 -0
  210. metadata +141 -63
  211. data/ext/prism/api_pack.c +0 -276
  212. data/include/prism/defines.h +0 -260
  213. data/include/prism/pack.h +0 -163
  214. data/include/prism/regexp.h +0 -43
  215. data/include/prism/static_literals.h +0 -121
  216. data/include/prism/util/pm_buffer.h +0 -236
  217. data/include/prism/util/pm_char.h +0 -204
  218. data/include/prism/util/pm_constant_pool.h +0 -218
  219. data/include/prism/util/pm_integer.h +0 -130
  220. data/include/prism/util/pm_memchr.h +0 -29
  221. data/include/prism/util/pm_newline_list.h +0 -113
  222. data/include/prism/util/pm_string.h +0 -200
  223. data/lib/prism/pack.rb +0 -230
  224. data/lib/prism/parse_result/comments.rb +0 -188
  225. data/rbi/prism/compiler.rbi +0 -12
  226. data/rbi/prism/dsl.rbi +0 -524
  227. data/rbi/prism/inspect_visitor.rbi +0 -12
  228. data/rbi/prism/node.rbi +0 -8750
  229. data/rbi/prism/node_ext.rbi +0 -107
  230. data/rbi/prism/parse_result.rbi +0 -404
  231. data/rbi/prism/reflection.rbi +0 -58
  232. data/rbi/prism/string_query.rbi +0 -12
  233. data/rbi/prism/visitor.rbi +0 -473
  234. data/rbi/prism.rbi +0 -63
  235. data/sig/prism/compiler.rbs +0 -9
  236. data/sig/prism/dispatcher.rbs +0 -19
  237. data/sig/prism/dot_visitor.rbs +0 -6
  238. data/sig/prism/dsl.rbs +0 -351
  239. data/sig/prism/inspect_visitor.rbs +0 -22
  240. data/sig/prism/lex_compat.rbs +0 -10
  241. data/sig/prism/node.rbs +0 -4042
  242. data/sig/prism/node_ext.rbs +0 -149
  243. data/sig/prism/pack.rbs +0 -43
  244. data/sig/prism/parse_result/comments.rbs +0 -38
  245. data/sig/prism/parse_result.rbs +0 -197
  246. data/sig/prism/pattern.rbs +0 -13
  247. data/sig/prism/reflection.rbs +0 -50
  248. data/sig/prism/relocation.rbs +0 -185
  249. data/sig/prism/serialize.rbs +0 -8
  250. data/sig/prism/string_query.rbs +0 -11
  251. data/sig/prism/visitor.rbs +0 -169
  252. data/sig/prism.rbs +0 -272
  253. data/src/pack.c +0 -509
  254. data/src/util/pm_constant_pool.c +0 -342
  255. data/src/util/pm_list.c +0 -49
  256. data/src/util/pm_newline_list.c +0 -125
  257. data/src/util/pm_string.c +0 -381
  258. data/src/util/pm_strpbrk.c +0 -206
@@ -4,6 +4,8 @@
4
4
  #include <ruby/win32.h>
5
5
  #endif
6
6
 
7
+ #include <errno.h>
8
+
7
9
  // NOTE: this file should contain only bindings. All non-trivial logic should be
8
10
  // in libprism so it can be shared its the various callers.
9
11
 
@@ -29,6 +31,7 @@ VALUE rb_cPrismCurrentVersionError;
29
31
 
30
32
  VALUE rb_cPrismDebugEncoding;
31
33
 
34
+ ID rb_id_option_attach_comments;
32
35
  ID rb_id_option_command_line;
33
36
  ID rb_id_option_encoding;
34
37
  ID rb_id_option_filepath;
@@ -37,14 +40,52 @@ ID rb_id_option_frozen_string_literal;
37
40
  ID rb_id_option_line;
38
41
  ID rb_id_option_main_script;
39
42
  ID rb_id_option_partial_script;
43
+ ID rb_id_option_raise_error;
40
44
  ID rb_id_option_scopes;
41
45
  ID rb_id_option_version;
46
+
42
47
  ID rb_id_source_for;
48
+
43
49
  ID rb_id_forwarding_positionals;
44
50
  ID rb_id_forwarding_keywords;
45
51
  ID rb_id_forwarding_block;
46
52
  ID rb_id_forwarding_all;
47
53
 
54
+ ID rb_id_raise_error_plain;
55
+ ID rb_id_raise_error_style;
56
+ ID rb_id_raise_error_color;
57
+
58
+ /******************************************************************************/
59
+ /* Result struct for working with functions that may need to raise errors. */
60
+ /******************************************************************************/
61
+
62
+ typedef struct {
63
+ enum {
64
+ RESULT_OK,
65
+ RESULT_ERR
66
+ } type;
67
+ VALUE value;
68
+ } result_t;
69
+
70
+ static result_t
71
+ result_ok(VALUE value) {
72
+ return (result_t) { .type = RESULT_OK, .value = value };
73
+ }
74
+
75
+ static result_t
76
+ result_err(VALUE value) {
77
+ return (result_t) { .type = RESULT_ERR, .value = value };
78
+ }
79
+
80
+ static VALUE
81
+ result_get(result_t result) {
82
+ if (result.type == RESULT_OK) {
83
+ return result.value;
84
+ } else {
85
+ rb_exc_raise(result.value);
86
+ }
87
+ }
88
+
48
89
  /******************************************************************************/
49
90
  /* IO of Ruby code */
50
91
  /******************************************************************************/
@@ -64,19 +105,6 @@ check_string(VALUE value) {
64
105
  return RSTRING_PTR(value);
65
106
  }
66
107
 
67
- /**
68
- * Load the contents and size of the given string into the given pm_string_t.
69
- */
70
- static void
71
- input_load_string(pm_string_t *input, VALUE string) {
72
- // Check if the string is a string. If it's not, then raise a type error.
73
- if (!RB_TYPE_P(string, T_STRING)) {
74
- rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(string));
75
- }
76
-
77
- pm_string_constant_init(input, RSTRING_PTR(string), RSTRING_LEN(string));
78
- }
79
-
80
108
  /******************************************************************************/
81
109
  /* Building C options from Ruby options */
82
110
  /******************************************************************************/
@@ -148,10 +176,8 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
148
176
 
149
177
  // Initialize the scope array.
150
178
  size_t locals_count = RARRAY_LEN(locals);
151
- pm_options_scope_t *options_scope = &options->scopes[scope_index];
152
- if (!pm_options_scope_init(options_scope, locals_count)) {
153
- rb_raise(rb_eNoMemError, "failed to allocate memory");
154
- }
179
+ pm_options_scope_t *options_scope = pm_options_scope_mut(options, scope_index);
180
+ pm_options_scope_init(options_scope, locals_count);
155
181
 
156
182
  // Iterate over the locals and add them to the scope.
157
183
  for (size_t local_index = 0; local_index < locals_count; local_index++) {
@@ -164,9 +190,9 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
164
190
  }
165
191
 
166
192
  // Add the local to the scope.
167
- pm_string_t *scope_local = &options_scope->locals[local_index];
193
+ pm_string_t *scope_local = pm_options_scope_local_mut(options_scope, local_index);
168
194
  const char *name = rb_id2name(SYM2ID(local));
169
- pm_string_constant_init(scope_local, name, strlen(name));
195
+ pm_string_init(scope_local, (const uint8_t *) name, strlen(name));
170
196
  }
171
197
 
172
198
  // Now set the forwarding options.
@@ -174,6 +200,23 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
174
200
  }
175
201
  }
176
202
 
203
+ /**
204
+ * Set the version option to the syntax version of the currently running Ruby,
205
+ * or to the closest version prism supports if it does not know that one.
206
+ */
207
+ static void
208
+ options_version_set_nearest(pm_options_t *options) {
209
+ if (!pm_options_version_set(options, ruby_version, 3)) {
210
+ // Prism doesn't know this specific version. Is it lower?
211
+ if (ruby_version[0] < '3' || (ruby_version[0] == '3' && ruby_version[2] < '3')) {
212
+ pm_options_version_set_lowest(options);
213
+ } else {
214
+ // Must be higher.
215
+ pm_options_version_set_highest(options);
216
+ }
217
+ }
218
+ }
219
+
177
220
  /**
178
221
  * An iterator function that is called for each key-value in the keywords hash.
179
222
  */
@@ -201,10 +244,11 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
201
244
  const char *version = check_string(value);
202
245
 
203
246
  if (RSTRING_LEN(value) == 7 && strncmp(version, "current", 7) == 0) {
204
- const char *current_version = RSTRING_PTR(rb_const_get(rb_cObject, rb_intern("RUBY_VERSION")));
205
- if (!pm_options_version_set(options, current_version, 3)) {
206
- rb_exc_raise(rb_exc_new_cstr(rb_cPrismCurrentVersionError, current_version));
247
+ if (!pm_options_version_set(options, ruby_version, 3)) {
248
+ rb_exc_raise(rb_exc_new_cstr(rb_cPrismCurrentVersionError, ruby_version));
207
249
  }
250
+ } else if (RSTRING_LEN(value) == 7 && strncmp(version, "nearest", 7) == 0) {
251
+ options_version_set_nearest(options);
208
252
  } else if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
209
253
  rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value);
210
254
  }
@@ -236,6 +280,46 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
236
280
  if (!NIL_P(value)) pm_options_partial_script_set(options, RTEST(value));
237
281
  } else if (key_id == rb_id_option_freeze) {
238
282
  if (!NIL_P(value)) pm_options_freeze_set(options, RTEST(value));
283
+ } else if (key_id == rb_id_option_attach_comments) {
284
+ if (!NIL_P(value)) pm_options_attach_comments_set(options, RTEST(value));
285
+ } else if (key_id == rb_id_option_raise_error) {
286
+ if (!NIL_P(value)) {
287
+ if (value == Qtrue) {
288
+ /* When given true, color when $stderr is a terminal (unless
289
+ * NO_COLOR is set), bold styling when NO_COLOR is set, plain
290
+ * otherwise.
291
+ *
292
+ * rb_stderr_tty_p is not available to extensions, so instead
293
+ * we call the tty? method on $stderr, guarding against it
294
+ * having been replaced by an object that does not respond to
295
+ * tty?. */
296
+ if (rb_respond_to(rb_stderr, rb_intern_const("tty?")) && RTEST(rb_funcall(rb_stderr, rb_intern_const("tty?"), 0))) {
297
+ const char *no_color = getenv("NO_COLOR");
298
+ if (no_color == NULL || no_color[0] == '\0') {
299
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_COLOR);
300
+ } else {
301
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_STYLE);
302
+ }
303
+ } else {
304
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_PLAIN);
305
+ }
306
+ } else {
307
+ if (!SYMBOL_P(value)) {
308
+ rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Symbol)", rb_obj_class(value));
309
+ }
310
+
311
+ ID value_id = SYM2ID(value);
312
+ if (value_id == rb_id_raise_error_plain) {
313
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_PLAIN);
314
+ } else if (value_id == rb_id_raise_error_style) {
315
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_STYLE);
316
+ } else if (value_id == rb_id_raise_error_color) {
317
+ pm_options_raise_error_set(options, (uint8_t) PM_ERRORS_FORMAT_COLOR);
318
+ } else {
319
+ rb_raise(rb_eArgError, "invalid raise_error value: %" PRIsVALUE, value);
320
+ }
321
+ }
322
+ }
239
323
  } else {
240
324
  rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, key);
241
325
  }
@@ -269,7 +353,8 @@ build_options(VALUE argument) {
269
353
  */
270
354
  static void
271
355
  extract_options(pm_options_t *options, VALUE filepath, VALUE keywords) {
272
- options->line = 1; // default
356
+ pm_options_line_set(options, 1); /* default */
357
+ options_version_set_nearest(options); /* default */
273
358
 
274
359
  if (!NIL_P(keywords)) {
275
360
  struct build_options_data data = { .options = options, .keywords = keywords };
@@ -297,36 +382,48 @@ extract_options(pm_options_t *options, VALUE filepath, VALUE keywords) {
297
382
  /**
298
383
  * Read options for methods that look like (source, **options).
299
384
  */
300
- static void
301
- string_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
385
+ static VALUE
386
+ string_options(int argc, VALUE *argv, pm_options_t *options) {
302
387
  VALUE string;
303
388
  VALUE keywords;
304
389
  rb_scan_args(argc, argv, "1:", &string, &keywords);
305
390
 
391
+ if (!RB_TYPE_P(string, T_STRING)) {
392
+ pm_options_free(options);
393
+ rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(string));
394
+ }
395
+
306
396
  extract_options(options, Qnil, keywords);
307
- input_load_string(input, string);
397
+ return string;
308
398
  }
309
399
 
310
400
  /**
311
401
  * Read options for methods that look like (filepath, **options).
312
402
  */
313
- static void
314
- file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options, VALUE *encoded_filepath) {
403
+ static pm_source_t *
404
+ file_options(int argc, VALUE *argv, pm_options_t *options, VALUE *encoded_filepath) {
315
405
  VALUE filepath;
316
406
  VALUE keywords;
317
407
  rb_scan_args(argc, argv, "1:", &filepath, &keywords);
318
408
 
319
- Check_Type(filepath, T_STRING);
409
+ int state = 0;
410
+ filepath = rb_protect(rb_get_path, filepath, &state);
411
+ if (state != 0) {
412
+ pm_options_free(options);
413
+ rb_jump_tag(state);
414
+ }
415
+
320
416
  *encoded_filepath = rb_str_encode_ospath(filepath);
321
417
  extract_options(options, *encoded_filepath, keywords);
322
418
 
323
- const char *source = (const char *) pm_string_source(&options->filepath);
324
- pm_string_init_result_t result;
419
+ const char *source = (const char *) pm_string_source(pm_options_filepath(options));
420
+ pm_source_init_result_t result;
421
+ pm_source_t *pm_src = pm_source_file_new(source, &result);
325
422
 
326
- switch (result = pm_string_file_init(input, source)) {
327
- case PM_STRING_INIT_SUCCESS:
423
+ switch (result) {
424
+ case PM_SOURCE_INIT_SUCCESS:
328
425
  break;
329
- case PM_STRING_INIT_ERROR_GENERIC: {
426
+ case PM_SOURCE_INIT_ERROR_GENERIC: {
330
427
  pm_options_free(options);
331
428
 
332
429
  #ifdef _WIN32
@@ -338,7 +435,7 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options, V
338
435
  rb_syserr_fail(e, source);
339
436
  break;
340
437
  }
341
- case PM_STRING_INIT_ERROR_DIRECTORY:
438
+ case PM_SOURCE_INIT_ERROR_DIRECTORY:
342
439
  pm_options_free(options);
343
440
  rb_syserr_fail(EISDIR, source);
344
441
  break;
@@ -347,6 +444,58 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options, V
347
444
  rb_raise(rb_eRuntimeError, "Unknown error (%d) initializing file: %s", result, source);
348
445
  break;
349
446
  }
447
+
448
+ return pm_src;
449
+ }
450
+
451
+ /**
452
+ * Check if the given parser has any errors and the raise option was set. If
453
+ * both are true, format them and return the formatted error wrapped in a
454
+ * result.
455
+ */
456
+ static result_t
457
+ check_raise_error_option(pm_parser_t *parser, const pm_options_t *options, rb_encoding *path_encoding) {
458
+ if (pm_parser_errors_size(parser) == 0) return result_ok(Qnil);
459
+
460
+ uint8_t raise_error = pm_options_raise_error(options);
461
+ if (raise_error == 0) return result_ok(Qnil);
462
+
463
+ pm_buffer_t *buffer = pm_buffer_new();
464
+ if (buffer == NULL) {
465
+ return result_err(rb_exc_new_cstr(rb_eNoMemError, "failed to allocate memory"));
466
+ }
467
+
468
+ pm_error_level_t error_level = pm_errors_format(parser, buffer, (pm_errors_format_type_t) raise_error);
469
+
470
+ /* Copy the message out of the buffer and free it before building any
471
+ * other Ruby objects so that the buffer cannot be leaked if building the
472
+ * objects raises. */
473
+ rb_encoding *message_encoding = (error_level == PM_ERROR_LEVEL_LOAD) ? rb_locale_encoding() : rb_enc_find(pm_parser_encoding_name(parser));
474
+ VALUE message = rb_enc_str_new(pm_buffer_value(buffer), (long) pm_buffer_length(buffer), message_encoding);
475
+ pm_buffer_free(buffer);
476
+
477
+ VALUE error = Qnil;
478
+ switch (error_level) {
479
+ case PM_ERROR_LEVEL_SYNTAX: {
480
+ error = rb_exc_new_str(rb_eSyntaxError, message);
481
+
482
+ rb_encoding *path_encoding_normal = path_encoding == NULL ? rb_utf8_encoding() : path_encoding;
483
+ VALUE path = rb_enc_str_new((const char *) pm_string_source(pm_options_filepath(options)), pm_string_length(pm_options_filepath(options)), path_encoding_normal);
484
+ rb_ivar_set(error, rb_intern_const("@path"), path);
485
+ break;
486
+ }
487
+ case PM_ERROR_LEVEL_ARGUMENT: {
488
+ error = rb_exc_new_str(rb_eArgError, message);
489
+ break;
490
+ }
491
+ case PM_ERROR_LEVEL_LOAD: {
492
+ error = rb_exc_new_str(rb_eLoadError, message);
493
+ rb_ivar_set(error, rb_intern_const("@path"), Qnil);
494
+ break;
495
+ }
496
+ }
497
+
498
+ return result_err(error);
350
499
  }
351
500
 
352
501
  #ifndef PRISM_EXCLUDE_SERIALIZATION
@@ -358,80 +507,88 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options, V
358
507
  /**
359
508
  * Dump the AST corresponding to the given input to a string.
360
509
  */
361
- static VALUE
362
- dump_input(pm_string_t *input, const pm_options_t *options) {
363
- pm_buffer_t buffer;
364
- if (!pm_buffer_init(&buffer)) {
365
- rb_raise(rb_eNoMemError, "failed to allocate memory");
510
+ static result_t
511
+ dump_input(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding) {
512
+ pm_arena_t *arena = pm_arena_new();
513
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
514
+ pm_node_t *node = pm_parse(parser);
515
+
516
+ result_t result = check_raise_error_option(parser, options, path_encoding);
517
+ if (result.type == RESULT_OK) {
518
+ pm_buffer_t *buffer = pm_buffer_new();
519
+ if (buffer) {
520
+ pm_serialize(parser, node, buffer);
521
+ result = result_ok(rb_str_new(pm_buffer_value(buffer), pm_buffer_length(buffer)));
522
+ pm_buffer_free(buffer);
523
+
524
+ if (pm_options_freeze(options)) rb_obj_freeze(result.value);
525
+ } else {
526
+ result = result_err(rb_exc_new_cstr(rb_eNoMemError, "failed to allocate memory"));
527
+ }
366
528
  }
367
529
 
368
- pm_parser_t parser;
369
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
370
-
371
- pm_node_t *node = pm_parse(&parser);
372
- pm_serialize(&parser, node, &buffer);
373
-
374
- VALUE result = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
375
- pm_node_destroy(&parser, node);
376
- pm_buffer_free(&buffer);
377
- pm_parser_free(&parser);
530
+ pm_parser_free(parser);
531
+ pm_arena_free(arena);
378
532
 
379
533
  return result;
380
534
  }
381
535
 
382
536
  /**
537
+ * :markup: markdown
383
538
  * call-seq:
384
- * Prism::dump(source, **options) -> String
539
+ * dump(source, **options) -> String
385
540
  *
386
541
  * Dump the AST corresponding to the given string to a string. For supported
387
- * options, see Prism::parse.
542
+ * options, see Prism.parse.
388
543
  */
389
544
  static VALUE
390
545
  dump(int argc, VALUE *argv, VALUE self) {
391
- pm_string_t input;
392
- pm_options_t options = { 0 };
393
- string_options(argc, argv, &input, &options);
546
+ pm_options_t *options = pm_options_new();
547
+ VALUE string = string_options(argc, argv, options);
548
+
549
+ const uint8_t *source = (const uint8_t *) RSTRING_PTR(string);
550
+ size_t length = RSTRING_LEN(string);
394
551
 
395
552
  #ifdef PRISM_BUILD_DEBUG
396
- size_t length = pm_string_length(&input);
397
553
  char* dup = xmalloc(length);
398
- memcpy(dup, pm_string_source(&input), length);
399
- pm_string_constant_init(&input, dup, length);
554
+ memcpy(dup, source, length);
555
+ source = (const uint8_t *) dup;
400
556
  #endif
401
557
 
402
- VALUE value = dump_input(&input, &options);
403
- if (options.freeze) rb_obj_freeze(value);
558
+ result_t result = dump_input(source, length, options, NULL);
404
559
 
405
560
  #ifdef PRISM_BUILD_DEBUG
561
+ #ifdef xfree_sized
562
+ xfree_sized(dup, length);
563
+ #else
406
564
  xfree(dup);
565
+ #endif
407
566
  #endif
408
567
 
409
- pm_string_free(&input);
410
- pm_options_free(&options);
411
-
412
- return value;
568
+ pm_options_free(options);
569
+ return result_get(result);
413
570
  }
414
571
 
415
572
  /**
573
+ * :markup: markdown
416
574
  * call-seq:
417
- * Prism::dump_file(filepath, **options) -> String
575
+ * dump_file(filepath, **options) -> String
418
576
  *
419
577
  * Dump the AST corresponding to the given file to a string. For supported
420
- * options, see Prism::parse.
578
+ * options, see Prism.parse.
421
579
  */
422
580
  static VALUE
423
581
  dump_file(int argc, VALUE *argv, VALUE self) {
424
- pm_string_t input;
425
- pm_options_t options = { 0 };
582
+ pm_options_t *options = pm_options_new();
426
583
 
427
584
  VALUE encoded_filepath;
428
- file_options(argc, argv, &input, &options, &encoded_filepath);
585
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
429
586
 
430
- VALUE value = dump_input(&input, &options);
431
- pm_string_free(&input);
432
- pm_options_free(&options);
587
+ result_t result = dump_input(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath));
588
+ pm_source_free(src);
589
+ pm_options_free(options);
433
590
 
434
- return value;
591
+ return result_get(result);
435
592
  }
436
593
 
437
594
  #endif
@@ -455,54 +612,54 @@ rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool free
455
612
  * Create a new Location instance from the given parser and bounds.
456
613
  */
457
614
  static inline VALUE
458
- parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint8_t *start, size_t length) {
459
- VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(length) };
615
+ parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) {
616
+ VALUE argv[] = { source, LONG2FIX(start), LONG2FIX(length) };
460
617
  return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze);
461
618
  }
462
619
 
463
620
  /**
464
621
  * Create a new Location instance from the given parser and location.
465
622
  */
466
- #define PARSER_LOCATION_LOC(parser, source, freeze, loc) \
467
- parser_location(parser, source, freeze, loc.start, (size_t) (loc.end - loc.start))
623
+ #define PARSER_LOCATION(source, freeze, location) \
624
+ parser_location(source, freeze, location.start, location.length)
468
625
 
469
626
  /**
470
627
  * Build a new Comment instance from the given parser and comment.
471
628
  */
472
629
  static inline VALUE
473
- parser_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_comment_t *comment) {
474
- VALUE argv[] = { PARSER_LOCATION_LOC(parser, source, freeze, comment->location) };
475
- VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
630
+ parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) {
631
+ VALUE argv[] = { PARSER_LOCATION(source, freeze, pm_comment_location(comment)) };
632
+ VALUE type = (pm_comment_type(comment) == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
476
633
  return rb_class_new_instance_freeze(1, argv, type, freeze);
477
634
  }
478
635
 
479
636
  /**
480
- * Extract the comments out of the parser into an array.
637
+ * Extract the comments out of the parser into the given array, which was
638
+ * created before the source so that the source could hold it. The caller may
639
+ * have already resized it to the number of comments, so each comment is stored
640
+ * at its index rather than pushed.
481
641
  */
482
- static VALUE
483
- parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
484
- VALUE comments = rb_ary_new_capa(parser->comment_list.size);
485
-
486
- for (
487
- const pm_comment_t *comment = (const pm_comment_t *) parser->comment_list.head;
488
- comment != NULL;
489
- comment = (const pm_comment_t *) comment->node.next
490
- ) {
491
- VALUE value = parser_comment(parser, source, freeze, comment);
492
- rb_ary_push(comments, value);
642
+ static void
643
+ parser_comments(const pm_parser_t *parser, VALUE comments, VALUE source, bool freeze) {
644
+ long index = 0;
645
+ for (const pm_comment_t *comment = pm_parser_comments_first(parser); comment != NULL; comment = pm_comment_next(comment)) {
646
+ rb_ary_store(comments, index++, parser_comment(source, freeze, comment));
493
647
  }
494
648
 
495
649
  if (freeze) rb_obj_freeze(comments);
496
- return comments;
497
650
  }
498
651
 
499
652
  /**
500
653
  * Build a new MagicComment instance from the given parser and magic comment.
501
654
  */
502
655
  static inline VALUE
503
- parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) {
504
- VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key_start, magic_comment->key_length);
505
- VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value_start, magic_comment->value_length);
656
+ parser_magic_comment(VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) {
657
+ pm_location_t key = pm_magic_comment_key(magic_comment);
658
+ pm_location_t value = pm_magic_comment_value(magic_comment);
659
+
660
+ VALUE key_loc = parser_location(source, freeze, key.start, key.length);
661
+ VALUE value_loc = parser_location(source, freeze, value.start, value.length);
662
+
506
663
  VALUE argv[] = { key_loc, value_loc };
507
664
  return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze);
508
665
  }
@@ -512,15 +669,10 @@ parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const
512
669
  */
513
670
  static VALUE
514
671
  parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
515
- VALUE magic_comments = rb_ary_new_capa(parser->magic_comment_list.size);
516
-
517
- for (
518
- const pm_magic_comment_t *magic_comment = (const pm_magic_comment_t *) parser->magic_comment_list.head;
519
- magic_comment != NULL;
520
- magic_comment = (const pm_magic_comment_t *) magic_comment->node.next
521
- ) {
522
- VALUE value = parser_magic_comment(parser, source, freeze, magic_comment);
523
- rb_ary_push(magic_comments, value);
672
+ VALUE magic_comments = rb_ary_new_capa(pm_parser_magic_comments_size(parser));
673
+
674
+ for (const pm_magic_comment_t *magic_comment = pm_parser_magic_comments_first(parser); magic_comment != NULL; magic_comment = pm_magic_comment_next(magic_comment)) {
675
+ rb_ary_push(magic_comments, parser_magic_comment(source, freeze, magic_comment));
524
676
  }
525
677
 
526
678
  if (freeze) rb_obj_freeze(magic_comments);
@@ -533,90 +685,148 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
533
685
  */
534
686
  static VALUE
535
687
  parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) {
536
- if (parser->data_loc.end == NULL) {
688
+ const pm_location_t *data_loc = pm_parser_data_loc(parser);
689
+
690
+ if (data_loc->length == 0) {
537
691
  return Qnil;
538
692
  } else {
539
- return PARSER_LOCATION_LOC(parser, source, freeze, parser->data_loc);
693
+ return parser_location(source, freeze, data_loc->start, data_loc->length);
540
694
  }
541
695
  }
542
696
 
697
+ /**
698
+ * Build a new ParseError instance from the given error.
699
+ */
700
+ static VALUE
701
+ parser_error(const pm_diagnostic_t *diagnostic, rb_encoding *encoding, VALUE source, bool freeze) {
702
+ VALUE type = ID2SYM(rb_intern(pm_diagnostic_type(diagnostic)));
703
+ VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), encoding));
704
+ VALUE location = PARSER_LOCATION(source, freeze, pm_diagnostic_location(diagnostic));
705
+
706
+ pm_error_level_t error_level = pm_diagnostic_error_level(diagnostic);
707
+ VALUE level = Qnil;
708
+
709
+ switch (error_level) {
710
+ case PM_ERROR_LEVEL_SYNTAX:
711
+ level = ID2SYM(rb_intern("syntax"));
712
+ break;
713
+ case PM_ERROR_LEVEL_ARGUMENT:
714
+ level = ID2SYM(rb_intern("argument"));
715
+ break;
716
+ case PM_ERROR_LEVEL_LOAD:
717
+ level = ID2SYM(rb_intern("load"));
718
+ break;
719
+ default:
720
+ rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error_level);
721
+ }
722
+
723
+ VALUE argv[] = { type, message, location, level };
724
+ return rb_class_new_instance_freeze(4, argv, rb_cPrismParseError, freeze);
725
+ }
726
+
543
727
  /**
544
728
  * Extract the errors out of the parser into an array.
545
729
  */
546
730
  static VALUE
547
731
  parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
548
- VALUE errors = rb_ary_new_capa(parser->error_list.size);
549
-
550
- for (
551
- const pm_diagnostic_t *error = (const pm_diagnostic_t *) parser->error_list.head;
552
- error != NULL;
553
- error = (const pm_diagnostic_t *) error->node.next
554
- ) {
555
- VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id)));
556
- VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding));
557
- VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, error->location);
558
-
559
- VALUE level = Qnil;
560
- switch (error->level) {
561
- case PM_ERROR_LEVEL_SYNTAX:
562
- level = ID2SYM(rb_intern("syntax"));
563
- break;
564
- case PM_ERROR_LEVEL_ARGUMENT:
565
- level = ID2SYM(rb_intern("argument"));
566
- break;
567
- case PM_ERROR_LEVEL_LOAD:
568
- level = ID2SYM(rb_intern("load"));
569
- break;
570
- default:
571
- rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level);
572
- }
732
+ VALUE errors = rb_ary_new_capa(pm_parser_errors_size(parser));
573
733
 
574
- VALUE argv[] = { type, message, location, level };
575
- VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseError, freeze);
576
- rb_ary_push(errors, value);
734
+ for (const pm_diagnostic_t *error = pm_parser_errors_first(parser); error != NULL; error = pm_diagnostic_next(error)) {
735
+ rb_ary_push(errors, parser_error(error, encoding, source, freeze));
577
736
  }
578
737
 
579
738
  if (freeze) rb_obj_freeze(errors);
580
739
  return errors;
581
740
  }
582
741
 
742
+ /**
743
+ * Build a new ParseWarning instance from the given warning.
744
+ */
745
+ static VALUE
746
+ parser_warning(const pm_diagnostic_t *diagnostic, rb_encoding *encoding, VALUE source, bool freeze) {
747
+ VALUE type = ID2SYM(rb_intern(pm_diagnostic_type(diagnostic)));
748
+ VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), encoding));
749
+ VALUE location = PARSER_LOCATION(source, freeze, pm_diagnostic_location(diagnostic));
750
+
751
+ pm_warning_level_t warning_level = pm_diagnostic_warning_level(diagnostic);
752
+ VALUE level = Qnil;
753
+
754
+ switch (warning_level) {
755
+ case PM_WARNING_LEVEL_DEFAULT:
756
+ level = ID2SYM(rb_intern("default"));
757
+ break;
758
+ case PM_WARNING_LEVEL_VERBOSE:
759
+ level = ID2SYM(rb_intern("verbose"));
760
+ break;
761
+ default:
762
+ rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, warning_level);
763
+ }
764
+
765
+ VALUE argv[] = { type, message, location, level };
766
+ return rb_class_new_instance_freeze(4, argv, rb_cPrismParseWarning, freeze);
767
+ }
768
+
583
769
  /**
584
770
  * Extract the warnings out of the parser into an array.
585
771
  */
586
772
  static VALUE
587
773
  parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
588
- VALUE warnings = rb_ary_new_capa(parser->warning_list.size);
589
-
590
- for (
591
- const pm_diagnostic_t *warning = (const pm_diagnostic_t *) parser->warning_list.head;
592
- warning != NULL;
593
- warning = (const pm_diagnostic_t *) warning->node.next
594
- ) {
595
- VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id)));
596
- VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding));
597
- VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, warning->location);
598
-
599
- VALUE level = Qnil;
600
- switch (warning->level) {
601
- case PM_WARNING_LEVEL_DEFAULT:
602
- level = ID2SYM(rb_intern("default"));
603
- break;
604
- case PM_WARNING_LEVEL_VERBOSE:
605
- level = ID2SYM(rb_intern("verbose"));
606
- break;
607
- default:
608
- rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, warning->level);
609
- }
774
+ VALUE warnings = rb_ary_new_capa(pm_parser_warnings_size(parser));
610
775
 
611
- VALUE argv[] = { type, message, location, level };
612
- VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseWarning, freeze);
613
- rb_ary_push(warnings, value);
776
+ for (const pm_diagnostic_t *warning = pm_parser_warnings_first(parser); warning != NULL; warning = pm_diagnostic_next(warning)) {
777
+ rb_ary_push(warnings, parser_warning(warning, encoding, source, freeze));
614
778
  }
615
779
 
616
780
  if (freeze) rb_obj_freeze(warnings);
617
781
  return warnings;
618
782
  }
619
783
 
784
+ /**
785
+ * Create the arrays that hold the comments attached to each node, sized for the
786
+ * parser's comment attachments. They stay empty unless the attach_comments
787
+ * option was set.
788
+ */
789
+ static void
790
+ parser_attachments_new(const pm_parser_t *parser, VALUE *attachments, VALUE *ranges) {
791
+ uint32_t ranges_size = pm_parser_comment_ranges_size(parser);
792
+ *attachments = rb_ary_new_capa(ranges_size == 0 ? 0 : (long) pm_parser_comments_size(parser));
793
+ *ranges = rb_ary_new_capa(ranges_size == 0 ? 0 : (long) ranges_size + 1);
794
+ }
795
+
796
+ /**
797
+ * Fill in the comments attached to each node. The attachments are the Comment
798
+ * objects in the order of the parser's comment attachments, which groups them
799
+ * by node in order of node id. The ranges are indexed by node id: for each node
800
+ * that owns comments, the index at which its comments begin is stored at its
801
+ * node id and the index at which they end is stored at the following node id.
802
+ */
803
+ static void
804
+ parser_attachments(const pm_parser_t *parser, VALUE comments, VALUE attachments, VALUE ranges, bool freeze) {
805
+ uint32_t ranges_size = pm_parser_comment_ranges_size(parser);
806
+
807
+ if (ranges_size > 0) {
808
+ long size = RARRAY_LEN(comments);
809
+
810
+ for (long index = 0; index < size; index++) {
811
+ rb_ary_store(attachments, index, RARRAY_AREF(comments, pm_parser_comment_attachment(parser, (uint32_t) index)));
812
+ }
813
+
814
+ for (uint32_t node_id = 0; node_id < ranges_size; node_id++) {
815
+ pm_comment_range_t range = pm_parser_comment_range(parser, node_id);
816
+
817
+ if (range.length > 0) {
818
+ rb_ary_store(ranges, (long) node_id, LONG2FIX(range.index));
819
+ rb_ary_store(ranges, (long) node_id + 1, LONG2FIX(range.index + range.length));
820
+ }
821
+ }
822
+ }
823
+
824
+ if (freeze) {
825
+ rb_obj_freeze(attachments);
826
+ rb_obj_freeze(ranges);
827
+ }
828
+ }
829
+
620
830
  /**
621
831
  * Create a new parse result from the given parser, value, encoding, and source.
622
832
  */
@@ -624,11 +834,11 @@ static VALUE
624
834
  parse_result_create(VALUE class, const pm_parser_t *parser, VALUE value, rb_encoding *encoding, VALUE source, bool freeze) {
625
835
  VALUE result_argv[] = {
626
836
  value,
627
- parser_comments(parser, source, freeze),
628
837
  parser_magic_comments(parser, source, freeze),
629
838
  parser_data_loc(parser, source, freeze),
630
839
  parser_errors(parser, encoding, source, freeze),
631
840
  parser_warnings(parser, encoding, source, freeze),
841
+ pm_parser_continuable(parser) ? Qtrue : Qfalse,
632
842
  source
633
843
  };
634
844
 
@@ -657,18 +867,11 @@ typedef struct {
657
867
  * onto the tokens array.
658
868
  */
659
869
  static void
660
- parse_lex_token(void *data, pm_parser_t *parser, pm_token_t *token) {
661
- parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data;
870
+ parse_lex_token(pm_parser_t *parser, pm_token_t *token, void *data) {
871
+ parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) data;
662
872
 
663
873
  VALUE value = pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source, parse_lex_data->freeze);
664
- VALUE yields = rb_assoc_new(value, INT2FIX(parser->lex_state));
665
-
666
- if (parse_lex_data->freeze) {
667
- rb_obj_freeze(value);
668
- rb_obj_freeze(yields);
669
- }
670
-
671
- rb_ary_push(parse_lex_data->tokens, yields);
874
+ rb_ary_push(parse_lex_data->tokens, value);
672
875
  }
673
876
 
674
877
  /**
@@ -678,8 +881,8 @@ parse_lex_token(void *data, pm_parser_t *parser, pm_token_t *token) {
678
881
  */
679
882
  static void
680
883
  parse_lex_encoding_changed_callback(pm_parser_t *parser) {
681
- parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data;
682
- parse_lex_data->encoding = rb_enc_find(parser->encoding->name);
884
+ parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) pm_parser_lex_callback_data(parser);
885
+ parse_lex_data->encoding = rb_enc_find(pm_parser_encoding_name(parser));
683
886
 
684
887
  // Since the encoding changed, we need to go back and change the encoding of
685
888
  // the tokens that were already lexed. This is only going to end up being
@@ -689,9 +892,7 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
689
892
  VALUE next_tokens = rb_ary_new();
690
893
 
691
894
  for (long index = 0; index < RARRAY_LEN(tokens); index++) {
692
- VALUE yields = rb_ary_entry(tokens, index);
693
- VALUE token = rb_ary_entry(yields, 0);
694
-
895
+ VALUE token = rb_ary_entry(tokens, index);
695
896
  VALUE value = rb_ivar_get(token, rb_intern("@value"));
696
897
  VALUE next_value = rb_str_dup(value);
697
898
 
@@ -702,18 +903,17 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
702
903
  parse_lex_data->source,
703
904
  rb_ivar_get(token, rb_intern("@type")),
704
905
  next_value,
705
- rb_ivar_get(token, rb_intern("@location"))
906
+ rb_ivar_get(token, rb_intern("@location")),
907
+ rb_ivar_get(token, rb_intern("@state")),
706
908
  };
707
909
 
708
- VALUE next_token = rb_class_new_instance(4, next_token_argv, rb_cPrismToken);
709
- VALUE next_yields = rb_assoc_new(next_token, rb_ary_entry(yields, 1));
910
+ VALUE next_token = rb_class_new_instance(5, next_token_argv, rb_cPrismToken);
710
911
 
711
912
  if (parse_lex_data->freeze) {
712
913
  rb_obj_freeze(next_token);
713
- rb_obj_freeze(next_yields);
714
914
  }
715
915
 
716
- rb_ary_push(next_tokens, next_yields);
916
+ rb_ary_push(next_tokens, next_token);
717
917
  }
718
918
 
719
919
  rb_ary_replace(parse_lex_data->tokens, next_tokens);
@@ -723,107 +923,122 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
723
923
  * Parse the given input and return a ParseResult containing just the tokens or
724
924
  * the nodes and tokens.
725
925
  */
726
- static VALUE
727
- parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nodes) {
728
- pm_parser_t parser;
729
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
730
- pm_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback);
731
-
732
- VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input));
733
- VALUE offsets = rb_ary_new_capa(parser.newline_list.size);
734
- VALUE source = rb_funcall(rb_cPrismSource, rb_id_source_for, 3, source_string, LONG2NUM(parser.start_line), offsets);
926
+ static result_t
927
+ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding, bool return_nodes) {
928
+ pm_arena_t *arena = pm_arena_new();
929
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
930
+ pm_parser_encoding_changed_callback_set(parser, parse_lex_encoding_changed_callback);
931
+
932
+ VALUE source_string = rb_str_new((const char *) input, input_length);
933
+ VALUE offsets = rb_ary_new_capa(pm_parser_line_offsets(parser)->size);
934
+ VALUE comments = rb_ary_new();
935
+ VALUE attachments = rb_ary_new();
936
+ VALUE ranges = rb_ary_new();
937
+ VALUE source = rb_funcall(rb_cPrismSource, rb_id_source_for, 6, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, attachments, ranges);
735
938
 
736
939
  parse_lex_data_t parse_lex_data = {
737
940
  .source = source,
738
941
  .tokens = rb_ary_new(),
739
- .encoding = rb_utf8_encoding(),
740
- .freeze = options->freeze,
942
+ .encoding = rb_enc_find(pm_parser_encoding_name(parser)),
943
+ .freeze = pm_options_freeze(options),
741
944
  };
742
945
 
743
946
  parse_lex_data_t *data = &parse_lex_data;
744
- pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
745
- .data = (void *) data,
746
- .callback = parse_lex_token,
747
- };
947
+ pm_parser_lex_callback_set(parser, parse_lex_token, data);
748
948
 
749
- parser.lex_callback = &lex_callback;
750
- pm_node_t *node = pm_parse(&parser);
949
+ pm_node_t *node = pm_parse(parser);
751
950
 
752
- // Here we need to update the Source object to have the correct
753
- // encoding for the source string and the correct newline offsets.
754
- // We do it here because we've already created the Source object and given
755
- // it over to all of the tokens, and both of these are only set after pm_parse().
756
- rb_encoding *encoding = rb_enc_find(parser.encoding->name);
757
- rb_enc_associate(source_string, encoding);
951
+ result_t result = check_raise_error_option(parser, options, path_encoding);
952
+ if (result.type == RESULT_OK) {
953
+ /* Update the Source object with the correct encoding, line offsets,
954
+ * and source hash, which are only available after pm_parse()
955
+ * completes. */
956
+ rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
957
+ rb_enc_associate(source_string, encoding);
758
958
 
759
- for (size_t index = 0; index < parser.newline_list.size; index++) {
760
- rb_ary_push(offsets, ULONG2NUM(parser.newline_list.offsets[index]));
761
- }
959
+ const pm_line_offset_list_t *line_offsets = pm_parser_line_offsets(parser);
960
+ for (size_t index = 0; index < line_offsets->size; index++) {
961
+ rb_ary_store(offsets, (long) index, ULONG2NUM(line_offsets->offsets[index]));
962
+ }
762
963
 
763
- if (options->freeze) {
764
- rb_obj_freeze(source_string);
765
- rb_obj_freeze(offsets);
766
- rb_obj_freeze(source);
767
- rb_obj_freeze(parse_lex_data.tokens);
768
- }
964
+ rb_funcall(source, rb_intern("replace_checksum"), 1, ULL2NUM(pm_parser_checksum(parser)));
769
965
 
770
- VALUE result;
771
- if (return_nodes) {
772
- VALUE value = rb_ary_new_capa(2);
773
- rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding, source, options->freeze));
774
- rb_ary_push(value, parse_lex_data.tokens);
775
- if (options->freeze) rb_obj_freeze(value);
776
- result = parse_result_create(rb_cPrismParseLexResult, &parser, value, parse_lex_data.encoding, source, options->freeze);
777
- } else {
778
- result = parse_result_create(rb_cPrismLexResult, &parser, parse_lex_data.tokens, parse_lex_data.encoding, source, options->freeze);
966
+ if (pm_options_freeze(options)) {
967
+ rb_obj_freeze(source_string);
968
+ rb_obj_freeze(offsets);
969
+ rb_obj_freeze(source);
970
+ rb_obj_freeze(parse_lex_data.tokens);
971
+ }
972
+
973
+ /* The comment lists were created along with the source before parsing,
974
+ * so they are grown to their final sizes now that those are known. */
975
+ uint32_t ranges_size = pm_parser_comment_ranges_size(parser);
976
+ rb_ary_resize(comments, (long) pm_parser_comments_size(parser));
977
+
978
+ if (ranges_size > 0) {
979
+ rb_ary_resize(attachments, (long) pm_parser_comments_size(parser));
980
+ rb_ary_resize(ranges, (long) ranges_size + 1);
981
+ }
982
+
983
+ parser_comments(parser, comments, source, pm_options_freeze(options));
984
+ parser_attachments(parser, comments, attachments, ranges, pm_options_freeze(options));
985
+
986
+ if (return_nodes) {
987
+ VALUE value = rb_ary_new_capa(2);
988
+ rb_ary_push(value, pm_ast_new(parser, node, parse_lex_data.encoding, source, pm_options_freeze(options)));
989
+ rb_ary_push(value, parse_lex_data.tokens);
990
+ if (pm_options_freeze(options)) rb_obj_freeze(value);
991
+ result = result_ok(parse_result_create(rb_cPrismParseLexResult, parser, value, parse_lex_data.encoding, source, pm_options_freeze(options)));
992
+ } else {
993
+ result = result_ok(parse_result_create(rb_cPrismLexResult, parser, parse_lex_data.tokens, parse_lex_data.encoding, source, pm_options_freeze(options)));
994
+ }
779
995
  }
780
996
 
781
- pm_node_destroy(&parser, node);
782
- pm_parser_free(&parser);
997
+ pm_parser_free(parser);
998
+ pm_arena_free(arena);
783
999
 
784
1000
  return result;
785
1001
  }
786
1002
 
787
1003
  /**
1004
+ * :markup: markdown
788
1005
  * call-seq:
789
- * Prism::lex(source, **options) -> LexResult
1006
+ * lex(source, **options) -> LexResult
790
1007
  *
791
1008
  * Return a LexResult instance that contains an array of Token instances
792
- * corresponding to the given string. For supported options, see Prism::parse.
1009
+ * corresponding to the given string. For supported options, see Prism.parse.
793
1010
  */
794
1011
  static VALUE
795
1012
  lex(int argc, VALUE *argv, VALUE self) {
796
- pm_string_t input;
797
- pm_options_t options = { 0 };
798
- string_options(argc, argv, &input, &options);
1013
+ pm_options_t *options = pm_options_new();
1014
+ VALUE string = string_options(argc, argv, options);
799
1015
 
800
- VALUE result = parse_lex_input(&input, &options, false);
801
- pm_string_free(&input);
802
- pm_options_free(&options);
1016
+ result_t result = parse_lex_input((const uint8_t *) RSTRING_PTR(string), RSTRING_LEN(string), options, NULL, false);
1017
+ pm_options_free(options);
803
1018
 
804
- return result;
1019
+ return result_get(result);
805
1020
  }
806
1021
 
807
1022
  /**
1023
+ * :markup: markdown
808
1024
  * call-seq:
809
- * Prism::lex_file(filepath, **options) -> LexResult
1025
+ * lex_file(filepath, **options) -> LexResult
810
1026
  *
811
1027
  * Return a LexResult instance that contains an array of Token instances
812
- * corresponding to the given file. For supported options, see Prism::parse.
1028
+ * corresponding to the given file. For supported options, see Prism.parse.
813
1029
  */
814
1030
  static VALUE
815
1031
  lex_file(int argc, VALUE *argv, VALUE self) {
816
- pm_string_t input;
817
- pm_options_t options = { 0 };
1032
+ pm_options_t *options = pm_options_new();
818
1033
 
819
1034
  VALUE encoded_filepath;
820
- file_options(argc, argv, &input, &options, &encoded_filepath);
1035
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
821
1036
 
822
- VALUE value = parse_lex_input(&input, &options, false);
823
- pm_string_free(&input);
824
- pm_options_free(&options);
1037
+ result_t result = parse_lex_input(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath), false);
1038
+ pm_source_free(src);
1039
+ pm_options_free(options);
825
1040
 
826
- return value;
1041
+ return result_get(result);
827
1042
  }
828
1043
 
829
1044
  /******************************************************************************/
@@ -833,35 +1048,47 @@ lex_file(int argc, VALUE *argv, VALUE self) {
833
1048
  /**
834
1049
  * Parse the given input and return a ParseResult instance.
835
1050
  */
836
- static VALUE
837
- parse_input(pm_string_t *input, const pm_options_t *options) {
838
- pm_parser_t parser;
839
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
1051
+ static result_t
1052
+ parse_input(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding) {
1053
+ pm_arena_t *arena = pm_arena_new();
1054
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
1055
+
1056
+ pm_node_t *node = pm_parse(parser);
1057
+
1058
+ result_t result = check_raise_error_option(parser, options, path_encoding);
1059
+ if (result.type == RESULT_OK) {
1060
+ rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
840
1061
 
841
- pm_node_t *node = pm_parse(&parser);
842
- rb_encoding *encoding = rb_enc_find(parser.encoding->name);
1062
+ bool freeze = pm_options_freeze(options);
1063
+ VALUE comments = rb_ary_new_capa(pm_parser_comments_size(parser));
1064
+ VALUE attachments, ranges;
1065
+ parser_attachments_new(parser, &attachments, &ranges);
843
1066
 
844
- VALUE source = pm_source_new(&parser, encoding, options->freeze);
845
- VALUE value = pm_ast_new(&parser, node, encoding, source, options->freeze);
846
- VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source, options->freeze);
1067
+ VALUE source = pm_source_new(parser, encoding, comments, attachments, ranges, freeze);
1068
+ parser_comments(parser, comments, source, freeze);
1069
+ parser_attachments(parser, comments, attachments, ranges, freeze);
847
1070
 
848
- if (options->freeze) {
849
- rb_obj_freeze(source);
1071
+ VALUE value = pm_ast_new(parser, node, encoding, source, freeze);
1072
+ result = result_ok(parse_result_create(rb_cPrismParseResult, parser, value, encoding, source, freeze));
850
1073
  }
851
1074
 
852
- pm_node_destroy(&parser, node);
853
- pm_parser_free(&parser);
1075
+ pm_parser_free(parser);
1076
+ pm_arena_free(arena);
854
1077
 
855
1078
  return result;
856
1079
  }
857
1080
 
858
1081
  /**
1082
+ * :markup: markdown
859
1083
  * call-seq:
860
- * Prism::parse(source, **options) -> ParseResult
1084
+ * parse(source, **options) -> ParseResult
861
1085
  *
862
1086
  * Parse the given string and return a ParseResult instance. The options that
863
1087
  * are supported are:
864
1088
  *
1089
+ * * `attach_comments` - whether or not to attach each comment to the node that
1090
+ * owns it. When set, the comments on the result are sorted by start
1091
+ * offset. This should be a boolean or nil.
865
1092
  * * `command_line` - either nil or a string of the various options that were
866
1093
  * set on the command line. Valid values are combinations of "a", "l",
867
1094
  * "n", "p", and "x".
@@ -886,116 +1113,143 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
886
1113
  * script that you know will be embedded inside another script later, but
887
1114
  * you do not have that context yet. For example, when parsing an ERB
888
1115
  * template that will be evaluated inside another script.
1116
+ * * `raise_error` - either nil, true, or a symbol indicating that an error
1117
+ * should be raised if the source contains any errors. The message of
1118
+ * the error will include the source of the lines that contain errors
1119
+ * when possible, and the value of this option controls how those lines
1120
+ * are formatted. Valid values are `:plain` (no formatting), `:style`
1121
+ * (bold formatting), and `:color` (bold and color formatting). When
1122
+ * `true` is given, the formatting is determined by whether or not
1123
+ * `$stderr` is a terminal and whether or not the `NO_COLOR` environment
1124
+ * variable is set, mirroring the behavior of CRuby itself. Syntax-level
1125
+ * errors raise SyntaxError, argument-level errors raise ArgumentError,
1126
+ * and load-level errors raise LoadError. Note that this option is
1127
+ * honored by every API that accepts these options, including predicates
1128
+ * like Prism.parse_success?, which will raise instead of returning a
1129
+ * boolean when the source contains errors.
889
1130
  * * `scopes` - the locals that are in scope surrounding the code that is being
890
1131
  * parsed. This should be an array of arrays of symbols or nil. Scopes are
891
1132
  * ordered from the outermost scope to the innermost one.
892
1133
  * * `version` - the version of Ruby syntax that prism should used to parse Ruby
893
- * code. By default prism assumes you want to parse with the latest
894
- * version of Ruby syntax (which you can trigger with `nil` or
895
- * `"latest"`). You may also restrict the syntax to a specific version of
896
- * Ruby, e.g., with `"3.3.0"`. To parse with the same syntax version that
897
- * the current Ruby is running use `version: "current"`. Raises
898
- * ArgumentError if the version is not currently supported by Prism.
1134
+ * code. By default prism parses with the syntax version nearest to the
1135
+ * current Ruby that is running (which you can trigger with `nil` or
1136
+ * `"nearest"`): the running version if prism supports it, otherwise the
1137
+ * closest version that it does support. You may also restrict the syntax
1138
+ * to a specific version of Ruby, e.g., with `"3.3.0"`. To parse with
1139
+ * exactly the same syntax version that the current Ruby is running use
1140
+ * `version: "current"`, which raises Prism::CurrentVersionError if prism
1141
+ * does not support it. To parse with the latest version of Ruby syntax
1142
+ * that prism supports, use `version: "latest"`. Raises ArgumentError if
1143
+ * the version is not currently supported by Prism.
899
1144
  */
900
1145
  static VALUE
901
1146
  parse(int argc, VALUE *argv, VALUE self) {
902
- pm_string_t input;
903
- pm_options_t options = { 0 };
904
- string_options(argc, argv, &input, &options);
1147
+ pm_options_t *options = pm_options_new();
1148
+ VALUE string = string_options(argc, argv, options);
1149
+
1150
+ const uint8_t *source = (const uint8_t *) RSTRING_PTR(string);
1151
+ size_t length = RSTRING_LEN(string);
905
1152
 
906
1153
  #ifdef PRISM_BUILD_DEBUG
907
- size_t length = pm_string_length(&input);
908
1154
  char* dup = xmalloc(length);
909
- memcpy(dup, pm_string_source(&input), length);
910
- pm_string_constant_init(&input, dup, length);
1155
+ memcpy(dup, source, length);
1156
+ source = (const uint8_t *) dup;
911
1157
  #endif
912
1158
 
913
- VALUE value = parse_input(&input, &options);
1159
+ result_t result = parse_input(source, length, options, NULL);
914
1160
 
915
1161
  #ifdef PRISM_BUILD_DEBUG
1162
+ #ifdef xfree_sized
1163
+ xfree_sized(dup, length);
1164
+ #else
916
1165
  xfree(dup);
1166
+ #endif
917
1167
  #endif
918
1168
 
919
- pm_string_free(&input);
920
- pm_options_free(&options);
921
- return value;
1169
+ pm_options_free(options);
1170
+ return result_get(result);
922
1171
  }
923
1172
 
924
1173
  /**
1174
+ * :markup: markdown
925
1175
  * call-seq:
926
- * Prism::parse_file(filepath, **options) -> ParseResult
1176
+ * parse_file(filepath, **options) -> ParseResult
927
1177
  *
928
1178
  * Parse the given file and return a ParseResult instance. For supported
929
- * options, see Prism::parse.
1179
+ * options, see Prism.parse.
930
1180
  */
931
1181
  static VALUE
932
1182
  parse_file(int argc, VALUE *argv, VALUE self) {
933
- pm_string_t input;
934
- pm_options_t options = { 0 };
1183
+ pm_options_t *options = pm_options_new();
935
1184
 
936
1185
  VALUE encoded_filepath;
937
- file_options(argc, argv, &input, &options, &encoded_filepath);
1186
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
938
1187
 
939
- VALUE value = parse_input(&input, &options);
940
- pm_string_free(&input);
941
- pm_options_free(&options);
1188
+ result_t result = parse_input(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath));
1189
+ pm_source_free(src);
1190
+ pm_options_free(options);
942
1191
 
943
- return value;
1192
+ return result_get(result);
944
1193
  }
945
1194
 
946
1195
  /**
947
1196
  * Parse the given input and return nothing.
948
1197
  */
949
- static void
950
- profile_input(pm_string_t *input, const pm_options_t *options) {
951
- pm_parser_t parser;
952
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
1198
+ static result_t
1199
+ profile_input(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding) {
1200
+ pm_arena_t *arena = pm_arena_new();
1201
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
1202
+
1203
+ pm_parse(parser);
953
1204
 
954
- pm_node_t *node = pm_parse(&parser);
955
- pm_node_destroy(&parser, node);
956
- pm_parser_free(&parser);
1205
+ result_t result = check_raise_error_option(parser, options, path_encoding);
1206
+ pm_parser_free(parser);
1207
+ pm_arena_free(arena);
1208
+
1209
+ return result;
957
1210
  }
958
1211
 
959
1212
  /**
1213
+ * :markup: markdown
960
1214
  * call-seq:
961
- * Prism::profile(source, **options) -> nil
1215
+ * profile(source, **options) -> nil
962
1216
  *
963
1217
  * Parse the given string and return nothing. This method is meant to allow
964
1218
  * profilers to avoid the overhead of reifying the AST to Ruby. For supported
965
- * options, see Prism::parse.
1219
+ * options, see Prism.parse.
966
1220
  */
967
1221
  static VALUE
968
1222
  profile(int argc, VALUE *argv, VALUE self) {
969
- pm_string_t input;
970
- pm_options_t options = { 0 };
1223
+ pm_options_t *options = pm_options_new();
1224
+ VALUE string = string_options(argc, argv, options);
971
1225
 
972
- string_options(argc, argv, &input, &options);
973
- profile_input(&input, &options);
974
- pm_string_free(&input);
975
- pm_options_free(&options);
1226
+ result_t result = profile_input((const uint8_t *) RSTRING_PTR(string), RSTRING_LEN(string), options, NULL);
1227
+ pm_options_free(options);
1228
+ result_get(result);
976
1229
 
977
1230
  return Qnil;
978
1231
  }
979
1232
 
980
1233
  /**
1234
+ * :markup: markdown
981
1235
  * call-seq:
982
- * Prism::profile_file(filepath, **options) -> nil
1236
+ * profile_file(filepath, **options) -> nil
983
1237
  *
984
1238
  * Parse the given file and return nothing. This method is meant to allow
985
1239
  * profilers to avoid the overhead of reifying the AST to Ruby. For supported
986
- * options, see Prism::parse.
1240
+ * options, see Prism.parse.
987
1241
  */
988
1242
  static VALUE
989
1243
  profile_file(int argc, VALUE *argv, VALUE self) {
990
- pm_string_t input;
991
- pm_options_t options = { 0 };
1244
+ pm_options_t *options = pm_options_new();
992
1245
 
993
1246
  VALUE encoded_filepath;
994
- file_options(argc, argv, &input, &options, &encoded_filepath);
1247
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
995
1248
 
996
- profile_input(&input, &options);
997
- pm_string_free(&input);
998
- pm_options_free(&options);
1249
+ result_t result = profile_input(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath));
1250
+ pm_source_free(src);
1251
+ pm_options_free(options);
1252
+ result_get(result);
999
1253
 
1000
1254
  return Qnil;
1001
1255
  }
@@ -1009,13 +1263,40 @@ parse_stream_eof(void *stream) {
1009
1263
  }
1010
1264
 
1011
1265
  /**
1012
- * An implementation of fgets that is suitable for use with Ruby IO objects.
1266
+ * The largest number of bytes a single character can occupy in any encoding
1267
+ * that Ruby supports (CESU-8). `IO#gets(limit)` treats its argument as a soft
1268
+ * limit: to avoid splitting a multi-byte character it may return up to
1269
+ * `MAX_ENC_LEN - 1` more bytes than requested. Reserving `MAX_ENC_LEN` bytes of
1270
+ * headroom therefore leaves room for both that overshoot and the terminating
1271
+ * NUL byte.
1272
+ *
1273
+ * The value can be re-derived from the Ruby source tree with:
1274
+ *
1275
+ * grep -Erh "max (enc|byte) length" enc | grep -Eo '[0-9]+' | sort | tail -n1
1276
+ */
1277
+ #define MAX_ENC_LEN 6
1278
+
1279
+ /**
1280
+ * An implementation of fgets that is suitable for use with Ruby IO objects. As
1281
+ * required by pm_source_stream_fgets_t, this never writes more than `size`
1282
+ * bytes into `string` (including the terminating NUL byte).
1013
1283
  */
1014
1284
  static char *
1015
1285
  parse_stream_fgets(char *string, int size, void *stream) {
1016
- RUBY_ASSERT(size > 0);
1286
+ RUBY_ASSERT(size > MAX_ENC_LEN);
1017
1287
 
1018
- VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - 1));
1288
+ /*
1289
+ * Request fewer bytes than the buffer can hold. `gets` may return more
1290
+ * bytes than requested when the limit falls in the middle of a multi-byte
1291
+ * character, and the reserved headroom guarantees the result still fits.
1292
+ */
1293
+ VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - MAX_ENC_LEN));
1294
+
1295
+ /*
1296
+ * A well-behaved stream returns a String, or nil at EOF. Coerce anything
1297
+ * else to nil so that we never treat a non-String as a byte buffer.
1298
+ */
1299
+ line = rb_check_string_type(line);
1019
1300
  if (NIL_P(line)) {
1020
1301
  return NULL;
1021
1302
  }
@@ -1023,18 +1304,30 @@ parse_stream_fgets(char *string, int size, void *stream) {
1023
1304
  const char *cstr = RSTRING_PTR(line);
1024
1305
  long length = RSTRING_LEN(line);
1025
1306
 
1026
- memcpy(string, cstr, length);
1307
+ /*
1308
+ * Defensively clamp the copy. A misbehaving `gets` may ignore the limit
1309
+ * entirely and return an arbitrarily long string; we must never write past
1310
+ * the caller's buffer. One byte is reserved for the NUL terminator.
1311
+ */
1312
+ if (length > (long) (size - 1)) {
1313
+ length = (long) (size - 1);
1314
+ }
1315
+
1316
+ memcpy(string, cstr, (size_t) length);
1027
1317
  string[length] = '\0';
1028
1318
 
1029
1319
  return string;
1030
1320
  }
1031
1321
 
1322
+ #undef MAX_ENC_LEN
1323
+
1032
1324
  /**
1325
+ * :markup: markdown
1033
1326
  * call-seq:
1034
- * Prism::parse_stream(stream, **options) -> ParseResult
1327
+ * parse_stream(stream, **options) -> ParseResult
1035
1328
  *
1036
1329
  * Parse the given object that responds to `gets` and return a ParseResult
1037
- * instance. The options that are supported are the same as Prism::parse.
1330
+ * instance. The options that are supported are the same as Prism.parse.
1038
1331
  */
1039
1332
  static VALUE
1040
1333
  parse_stream(int argc, VALUE *argv, VALUE self) {
@@ -1042,187 +1335,213 @@ parse_stream(int argc, VALUE *argv, VALUE self) {
1042
1335
  VALUE keywords;
1043
1336
  rb_scan_args(argc, argv, "1:", &stream, &keywords);
1044
1337
 
1045
- pm_options_t options = { 0 };
1046
- extract_options(&options, Qnil, keywords);
1338
+ pm_options_t *options = pm_options_new();
1339
+ extract_options(options, Qnil, keywords);
1047
1340
 
1048
- pm_parser_t parser;
1049
- pm_buffer_t buffer;
1341
+ pm_source_t *src = pm_source_stream_new((void *) stream, parse_stream_fgets, parse_stream_eof);
1342
+ pm_arena_t *arena = pm_arena_new();
1343
+ pm_parser_t *parser;
1050
1344
 
1051
- pm_node_t *node = pm_parse_stream(&parser, &buffer, (void *) stream, parse_stream_fgets, parse_stream_eof, &options);
1052
- rb_encoding *encoding = rb_enc_find(parser.encoding->name);
1345
+ pm_node_t *node = pm_parse_stream(&parser, arena, src, options);
1053
1346
 
1054
- VALUE source = pm_source_new(&parser, encoding, options.freeze);
1055
- VALUE value = pm_ast_new(&parser, node, encoding, source, options.freeze);
1056
- VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source, options.freeze);
1347
+ result_t result = check_raise_error_option(parser, options, NULL);
1348
+ if (result.type == RESULT_OK) {
1349
+ rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
1057
1350
 
1058
- pm_node_destroy(&parser, node);
1059
- pm_buffer_free(&buffer);
1060
- pm_parser_free(&parser);
1351
+ VALUE comments = rb_ary_new_capa(pm_parser_comments_size(parser));
1352
+ VALUE attachments, ranges;
1353
+ parser_attachments_new(parser, &attachments, &ranges);
1061
1354
 
1062
- return result;
1355
+ VALUE source = pm_source_new(parser, encoding, comments, attachments, ranges, pm_options_freeze(options));
1356
+ parser_comments(parser, comments, source, pm_options_freeze(options));
1357
+ parser_attachments(parser, comments, attachments, ranges, pm_options_freeze(options));
1358
+
1359
+ VALUE value = pm_ast_new(parser, node, encoding, source, pm_options_freeze(options));
1360
+
1361
+ result = result_ok(parse_result_create(rb_cPrismParseResult, parser, value, encoding, source, pm_options_freeze(options)));
1362
+ }
1363
+
1364
+ pm_source_free(src);
1365
+ pm_parser_free(parser);
1366
+ pm_arena_free(arena);
1367
+ pm_options_free(options);
1368
+
1369
+ return result_get(result);
1063
1370
  }
1064
1371
 
1065
1372
  /**
1066
1373
  * Parse the given input and return an array of Comment objects.
1067
1374
  */
1068
- static VALUE
1069
- parse_input_comments(pm_string_t *input, const pm_options_t *options) {
1070
- pm_parser_t parser;
1071
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
1375
+ static result_t
1376
+ parse_input_comments(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding) {
1377
+ pm_arena_t *arena = pm_arena_new();
1378
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
1072
1379
 
1073
- pm_node_t *node = pm_parse(&parser);
1074
- rb_encoding *encoding = rb_enc_find(parser.encoding->name);
1380
+ pm_parse(parser);
1075
1381
 
1076
- VALUE source = pm_source_new(&parser, encoding, options->freeze);
1077
- VALUE comments = parser_comments(&parser, source, options->freeze);
1382
+ result_t result = check_raise_error_option(parser, options, path_encoding);
1383
+ if (result.type == RESULT_OK) {
1384
+ rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
1078
1385
 
1079
- pm_node_destroy(&parser, node);
1080
- pm_parser_free(&parser);
1386
+ VALUE comments = rb_ary_new_capa(pm_parser_comments_size(parser));
1387
+ VALUE attachments, ranges;
1388
+ parser_attachments_new(parser, &attachments, &ranges);
1081
1389
 
1082
- return comments;
1390
+ VALUE source = pm_source_new(parser, encoding, comments, attachments, ranges, pm_options_freeze(options));
1391
+ parser_comments(parser, comments, source, pm_options_freeze(options));
1392
+ parser_attachments(parser, comments, attachments, ranges, pm_options_freeze(options));
1393
+
1394
+ result = result_ok(comments);
1395
+ }
1396
+
1397
+ pm_parser_free(parser);
1398
+ pm_arena_free(arena);
1399
+
1400
+ return result;
1083
1401
  }
1084
1402
 
1085
1403
  /**
1404
+ * :markup: markdown
1086
1405
  * call-seq:
1087
- * Prism::parse_comments(source, **options) -> Array
1406
+ * parse_comments(source, **options) -> Array
1088
1407
  *
1089
1408
  * Parse the given string and return an array of Comment objects. For supported
1090
- * options, see Prism::parse.
1409
+ * options, see Prism.parse.
1091
1410
  */
1092
1411
  static VALUE
1093
1412
  parse_comments(int argc, VALUE *argv, VALUE self) {
1094
- pm_string_t input;
1095
- pm_options_t options = { 0 };
1096
- string_options(argc, argv, &input, &options);
1413
+ pm_options_t *options = pm_options_new();
1414
+ VALUE string = string_options(argc, argv, options);
1097
1415
 
1098
- VALUE result = parse_input_comments(&input, &options);
1099
- pm_string_free(&input);
1100
- pm_options_free(&options);
1416
+ result_t result = parse_input_comments((const uint8_t *) RSTRING_PTR(string), RSTRING_LEN(string), options, NULL);
1417
+ pm_options_free(options);
1101
1418
 
1102
- return result;
1419
+ return result_get(result);
1103
1420
  }
1104
1421
 
1105
1422
  /**
1423
+ * :markup: markdown
1106
1424
  * call-seq:
1107
- * Prism::parse_file_comments(filepath, **options) -> Array
1425
+ * parse_file_comments(filepath, **options) -> Array
1108
1426
  *
1109
1427
  * Parse the given file and return an array of Comment objects. For supported
1110
- * options, see Prism::parse.
1428
+ * options, see Prism.parse.
1111
1429
  */
1112
1430
  static VALUE
1113
1431
  parse_file_comments(int argc, VALUE *argv, VALUE self) {
1114
- pm_string_t input;
1115
- pm_options_t options = { 0 };
1432
+ pm_options_t *options = pm_options_new();
1116
1433
 
1117
1434
  VALUE encoded_filepath;
1118
- file_options(argc, argv, &input, &options, &encoded_filepath);
1435
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
1119
1436
 
1120
- VALUE value = parse_input_comments(&input, &options);
1121
- pm_string_free(&input);
1122
- pm_options_free(&options);
1437
+ result_t result = parse_input_comments(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath));
1438
+ pm_source_free(src);
1439
+ pm_options_free(options);
1123
1440
 
1124
- return value;
1441
+ return result_get(result);
1125
1442
  }
1126
1443
 
1127
1444
  /**
1445
+ * :markup: markdown
1128
1446
  * call-seq:
1129
- * Prism::parse_lex(source, **options) -> ParseLexResult
1447
+ * parse_lex(source, **options) -> ParseLexResult
1130
1448
  *
1131
1449
  * Parse the given string and return a ParseLexResult instance that contains a
1132
1450
  * 2-element array, where the first element is the AST and the second element is
1133
1451
  * an array of Token instances.
1134
1452
  *
1135
1453
  * This API is only meant to be used in the case where you need both the AST and
1136
- * the tokens. If you only need one or the other, use either Prism::parse or
1137
- * Prism::lex.
1454
+ * the tokens. If you only need one or the other, use either Prism.parse or
1455
+ * Prism.lex.
1138
1456
  *
1139
- * For supported options, see Prism::parse.
1457
+ * For supported options, see Prism.parse.
1140
1458
  */
1141
1459
  static VALUE
1142
1460
  parse_lex(int argc, VALUE *argv, VALUE self) {
1143
- pm_string_t input;
1144
- pm_options_t options = { 0 };
1145
- string_options(argc, argv, &input, &options);
1461
+ pm_options_t *options = pm_options_new();
1462
+ VALUE string = string_options(argc, argv, options);
1146
1463
 
1147
- VALUE value = parse_lex_input(&input, &options, true);
1148
- pm_string_free(&input);
1149
- pm_options_free(&options);
1464
+ result_t result = parse_lex_input((const uint8_t *) RSTRING_PTR(string), RSTRING_LEN(string), options, NULL, true);
1465
+ pm_options_free(options);
1150
1466
 
1151
- return value;
1467
+ return result_get(result);
1152
1468
  }
1153
1469
 
1154
1470
  /**
1471
+ * :markup: markdown
1155
1472
  * call-seq:
1156
- * Prism::parse_lex_file(filepath, **options) -> ParseLexResult
1473
+ * parse_lex_file(filepath, **options) -> ParseLexResult
1157
1474
  *
1158
1475
  * Parse the given file and return a ParseLexResult instance that contains a
1159
1476
  * 2-element array, where the first element is the AST and the second element is
1160
1477
  * an array of Token instances.
1161
1478
  *
1162
1479
  * This API is only meant to be used in the case where you need both the AST and
1163
- * the tokens. If you only need one or the other, use either Prism::parse_file
1164
- * or Prism::lex_file.
1480
+ * the tokens. If you only need one or the other, use either Prism.parse_file
1481
+ * or Prism.lex_file.
1165
1482
  *
1166
- * For supported options, see Prism::parse.
1483
+ * For supported options, see Prism.parse.
1167
1484
  */
1168
1485
  static VALUE
1169
1486
  parse_lex_file(int argc, VALUE *argv, VALUE self) {
1170
- pm_string_t input;
1171
- pm_options_t options = { 0 };
1487
+ pm_options_t *options = pm_options_new();
1172
1488
 
1173
1489
  VALUE encoded_filepath;
1174
- file_options(argc, argv, &input, &options, &encoded_filepath);
1490
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
1175
1491
 
1176
- VALUE value = parse_lex_input(&input, &options, true);
1177
- pm_string_free(&input);
1178
- pm_options_free(&options);
1492
+ result_t result = parse_lex_input(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath), true);
1493
+ pm_source_free(src);
1494
+ pm_options_free(options);
1179
1495
 
1180
- return value;
1496
+ return result_get(result);
1181
1497
  }
1182
1498
 
1183
1499
  /**
1184
1500
  * Parse the given input and return true if it parses without errors.
1185
1501
  */
1186
- static VALUE
1187
- parse_input_success_p(pm_string_t *input, const pm_options_t *options) {
1188
- pm_parser_t parser;
1189
- pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
1190
-
1191
- pm_node_t *node = pm_parse(&parser);
1192
- pm_node_destroy(&parser, node);
1502
+ static result_t
1503
+ parse_input_success_p(const uint8_t *input, size_t input_length, const pm_options_t *options, rb_encoding *path_encoding) {
1504
+ pm_arena_t *arena = pm_arena_new();
1505
+ pm_parser_t *parser = pm_parser_new(arena, input, input_length, options);
1506
+ pm_parse(parser);
1507
+
1508
+ result_t result = check_raise_error_option(parser, options, path_encoding);
1509
+ if (result.type == RESULT_OK) {
1510
+ result = result_ok(pm_parser_errors_size(parser) == 0 ? Qtrue : Qfalse);
1511
+ }
1193
1512
 
1194
- VALUE result = parser.error_list.size == 0 ? Qtrue : Qfalse;
1195
- pm_parser_free(&parser);
1513
+ pm_parser_free(parser);
1514
+ pm_arena_free(arena);
1196
1515
 
1197
1516
  return result;
1198
1517
  }
1199
1518
 
1200
1519
  /**
1520
+ * :markup: markdown
1201
1521
  * call-seq:
1202
- * Prism::parse_success?(source, **options) -> bool
1522
+ * parse_success?(source, **options) -> bool
1203
1523
  *
1204
1524
  * Parse the given string and return true if it parses without errors. For
1205
- * supported options, see Prism::parse.
1525
+ * supported options, see Prism.parse.
1206
1526
  */
1207
1527
  static VALUE
1208
1528
  parse_success_p(int argc, VALUE *argv, VALUE self) {
1209
- pm_string_t input;
1210
- pm_options_t options = { 0 };
1211
- string_options(argc, argv, &input, &options);
1529
+ pm_options_t *options = pm_options_new();
1530
+ VALUE string = string_options(argc, argv, options);
1212
1531
 
1213
- VALUE result = parse_input_success_p(&input, &options);
1214
- pm_string_free(&input);
1215
- pm_options_free(&options);
1532
+ result_t result = parse_input_success_p((const uint8_t *) RSTRING_PTR(string), RSTRING_LEN(string), options, NULL);
1533
+ pm_options_free(options);
1216
1534
 
1217
- return result;
1535
+ return result_get(result);
1218
1536
  }
1219
1537
 
1220
1538
  /**
1539
+ * :markup: markdown
1221
1540
  * call-seq:
1222
- * Prism::parse_failure?(source, **options) -> bool
1541
+ * parse_failure?(source, **options) -> bool
1223
1542
  *
1224
1543
  * Parse the given string and return true if it parses with errors. For
1225
- * supported options, see Prism::parse.
1544
+ * supported options, see Prism.parse.
1226
1545
  */
1227
1546
  static VALUE
1228
1547
  parse_failure_p(int argc, VALUE *argv, VALUE self) {
@@ -1230,33 +1549,34 @@ parse_failure_p(int argc, VALUE *argv, VALUE self) {
1230
1549
  }
1231
1550
 
1232
1551
  /**
1552
+ * :markup: markdown
1233
1553
  * call-seq:
1234
- * Prism::parse_file_success?(filepath, **options) -> bool
1554
+ * parse_file_success?(filepath, **options) -> bool
1235
1555
  *
1236
1556
  * Parse the given file and return true if it parses without errors. For
1237
- * supported options, see Prism::parse.
1557
+ * supported options, see Prism.parse.
1238
1558
  */
1239
1559
  static VALUE
1240
1560
  parse_file_success_p(int argc, VALUE *argv, VALUE self) {
1241
- pm_string_t input;
1242
- pm_options_t options = { 0 };
1561
+ pm_options_t *options = pm_options_new();
1243
1562
 
1244
1563
  VALUE encoded_filepath;
1245
- file_options(argc, argv, &input, &options, &encoded_filepath);
1564
+ pm_source_t *src = file_options(argc, argv, options, &encoded_filepath);
1246
1565
 
1247
- VALUE result = parse_input_success_p(&input, &options);
1248
- pm_string_free(&input);
1249
- pm_options_free(&options);
1566
+ result_t result = parse_input_success_p(pm_source_source(src), pm_source_length(src), options, rb_enc_get(encoded_filepath));
1567
+ pm_source_free(src);
1568
+ pm_options_free(options);
1250
1569
 
1251
- return result;
1570
+ return result_get(result);
1252
1571
  }
1253
1572
 
1254
1573
  /**
1574
+ * :markup: markdown
1255
1575
  * call-seq:
1256
- * Prism::parse_file_failure?(filepath, **options) -> bool
1576
+ * parse_file_failure?(filepath, **options) -> bool
1257
1577
  *
1258
1578
  * Parse the given file and return true if it parses with errors. For
1259
- * supported options, see Prism::parse.
1579
+ * supported options, see Prism.parse.
1260
1580
  */
1261
1581
  static VALUE
1262
1582
  parse_file_failure_p(int argc, VALUE *argv, VALUE self) {
@@ -1286,8 +1606,9 @@ string_query(pm_string_query_t result) {
1286
1606
  }
1287
1607
 
1288
1608
  /**
1609
+ * :markup: markdown
1289
1610
  * call-seq:
1290
- * Prism::StringQuery::local?(string) -> bool
1611
+ * local?(string) -> bool
1291
1612
  *
1292
1613
  * Returns true if the string constitutes a valid local variable name. Note that
1293
1614
  * this means the names that can be set through Binding#local_variable_set, not
@@ -1300,8 +1621,9 @@ string_query_local_p(VALUE self, VALUE string) {
1300
1621
  }
1301
1622
 
1302
1623
  /**
1624
+ * :markup: markdown
1303
1625
  * call-seq:
1304
- * Prism::StringQuery::constant?(string) -> bool
1626
+ * constant?(string) -> bool
1305
1627
  *
1306
1628
  * Returns true if the string constitutes a valid constant name. Note that this
1307
1629
  * means the names that can be set through Module#const_set, not necessarily the
@@ -1314,8 +1636,9 @@ string_query_constant_p(VALUE self, VALUE string) {
1314
1636
  }
1315
1637
 
1316
1638
  /**
1639
+ * :markup: markdown
1317
1640
  * call-seq:
1318
- * Prism::StringQuery::method_name?(string) -> bool
1641
+ * method_name?(string) -> bool
1319
1642
  *
1320
1643
  * Returns true if the string constitutes a valid method name.
1321
1644
  */
@@ -1325,6 +1648,20 @@ string_query_method_name_p(VALUE self, VALUE string) {
1325
1648
  return string_query(pm_string_query_method_name(source, RSTRING_LEN(string), rb_enc_get(string)->name));
1326
1649
  }
1327
1650
 
1651
+ /**
1652
+ * :markup: markdown
1653
+ * call-seq:
1654
+ * keyword?(string) -> bool
1655
+ *
1656
+ * Returns true if the string is a valid keyword, such as `if`, `defined?`, or
1657
+ * `__FILE__`.
1658
+ */
1659
+ static VALUE
1660
+ string_query_keyword_p(VALUE self, VALUE string) {
1661
+ const uint8_t *source = (const uint8_t *) check_string(string);
1662
+ return string_query(pm_string_query_keyword(source, RSTRING_LEN(string), rb_enc_get(string)->name));
1663
+ }
1664
+
1328
1665
  /******************************************************************************/
1329
1666
  /* Initialization of the extension */
1330
1667
  /******************************************************************************/
@@ -1334,8 +1671,8 @@ string_query_method_name_p(VALUE self, VALUE string) {
1334
1671
  */
1335
1672
  RUBY_FUNC_EXPORTED void
1336
1673
  Init_prism(void) {
1337
- // Make sure that the prism library version matches the expected version.
1338
- // Otherwise something was compiled incorrectly.
1674
+ /* Make sure that the prism library version matches the expected version.
1675
+ * Otherwise something was compiled incorrectly. */
1339
1676
  if (strcmp(pm_version(), EXPECTED_PRISM_VERSION) != 0) {
1340
1677
  rb_raise(
1341
1678
  rb_eRuntimeError,
@@ -1346,12 +1683,10 @@ Init_prism(void) {
1346
1683
  }
1347
1684
 
1348
1685
  #ifdef HAVE_RB_EXT_RACTOR_SAFE
1349
- // Mark this extension as Ractor-safe.
1686
+ /* Mark this extension as Ractor-safe. */
1350
1687
  rb_ext_ractor_safe(true);
1351
1688
  #endif
1352
1689
 
1353
- // Grab up references to all of the constants that we're going to need to
1354
- // reference throughout this extension.
1355
1690
  rb_cPrism = rb_define_module("Prism");
1356
1691
  rb_cPrismNode = rb_define_class_under(rb_cPrism, "Node", rb_cObject);
1357
1692
  rb_cPrismSource = rb_define_class_under(rb_cPrism, "Source", rb_cObject);
@@ -1372,8 +1707,9 @@ Init_prism(void) {
1372
1707
 
1373
1708
  rb_cPrismCurrentVersionError = rb_const_get(rb_cPrism, rb_intern("CurrentVersionError"));
1374
1709
 
1375
- // Intern all of the IDs eagerly that we support so that we don't have to do
1376
- // it every time we parse.
1710
+ /* Intern all of the IDs eagerly that we support so that we do not have to
1711
+ * do it every time we parse. */
1712
+ rb_id_option_attach_comments = rb_intern_const("attach_comments");
1377
1713
  rb_id_option_command_line = rb_intern_const("command_line");
1378
1714
  rb_id_option_encoding = rb_intern_const("encoding");
1379
1715
  rb_id_option_filepath = rb_intern_const("filepath");
@@ -1382,20 +1718,26 @@ Init_prism(void) {
1382
1718
  rb_id_option_line = rb_intern_const("line");
1383
1719
  rb_id_option_main_script = rb_intern_const("main_script");
1384
1720
  rb_id_option_partial_script = rb_intern_const("partial_script");
1721
+ rb_id_option_raise_error = rb_intern_const("raise_error");
1385
1722
  rb_id_option_scopes = rb_intern_const("scopes");
1386
1723
  rb_id_option_version = rb_intern_const("version");
1724
+
1387
1725
  rb_id_source_for = rb_intern("for");
1726
+
1388
1727
  rb_id_forwarding_positionals = rb_intern("*");
1389
1728
  rb_id_forwarding_keywords = rb_intern("**");
1390
1729
  rb_id_forwarding_block = rb_intern("&");
1391
1730
  rb_id_forwarding_all = rb_intern("...");
1392
1731
 
1732
+ rb_id_raise_error_plain = rb_intern_const("plain");
1733
+ rb_id_raise_error_style = rb_intern_const("style");
1734
+ rb_id_raise_error_color = rb_intern_const("color");
1735
+
1393
1736
  /**
1394
1737
  * The version of the prism library.
1395
1738
  */
1396
1739
  rb_define_const(rb_cPrism, "VERSION", rb_str_freeze(rb_str_new_cstr(EXPECTED_PRISM_VERSION)));
1397
1740
 
1398
- // First, the functions that have to do with lexing and parsing.
1399
1741
  rb_define_singleton_method(rb_cPrism, "lex", lex, -1);
1400
1742
  rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1);
1401
1743
  rb_define_singleton_method(rb_cPrism, "parse", parse, -1);
@@ -1420,8 +1762,7 @@ Init_prism(void) {
1420
1762
  rb_define_singleton_method(rb_cPrismStringQuery, "local?", string_query_local_p, 1);
1421
1763
  rb_define_singleton_method(rb_cPrismStringQuery, "constant?", string_query_constant_p, 1);
1422
1764
  rb_define_singleton_method(rb_cPrismStringQuery, "method_name?", string_query_method_name_p, 1);
1765
+ rb_define_singleton_method(rb_cPrismStringQuery, "keyword?", string_query_keyword_p, 1);
1423
1766
 
1424
- // Next, initialize the other APIs.
1425
1767
  Init_prism_api_node();
1426
- Init_prism_pack();
1427
1768
  }