prism 0.17.1 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +60 -1
  3. data/Makefile +5 -5
  4. data/README.md +4 -3
  5. data/config.yml +214 -68
  6. data/docs/build_system.md +6 -6
  7. data/docs/building.md +10 -3
  8. data/docs/configuration.md +11 -9
  9. data/docs/encoding.md +92 -88
  10. data/docs/heredocs.md +1 -1
  11. data/docs/javascript.md +29 -1
  12. data/docs/local_variable_depth.md +229 -0
  13. data/docs/ruby_api.md +16 -0
  14. data/docs/serialization.md +18 -13
  15. data/ext/prism/api_node.c +411 -240
  16. data/ext/prism/extconf.rb +97 -127
  17. data/ext/prism/extension.c +97 -33
  18. data/ext/prism/extension.h +1 -1
  19. data/include/prism/ast.h +377 -159
  20. data/include/prism/defines.h +17 -0
  21. data/include/prism/diagnostic.h +38 -6
  22. data/include/prism/{enc/pm_encoding.h → encoding.h} +126 -64
  23. data/include/prism/options.h +2 -2
  24. data/include/prism/parser.h +62 -36
  25. data/include/prism/regexp.h +2 -2
  26. data/include/prism/util/pm_buffer.h +9 -1
  27. data/include/prism/util/pm_memchr.h +2 -2
  28. data/include/prism/util/pm_strpbrk.h +3 -3
  29. data/include/prism/version.h +3 -3
  30. data/include/prism.h +13 -15
  31. data/lib/prism/compiler.rb +15 -3
  32. data/lib/prism/debug.rb +13 -4
  33. data/lib/prism/desugar_compiler.rb +4 -3
  34. data/lib/prism/dispatcher.rb +70 -14
  35. data/lib/prism/dot_visitor.rb +4612 -0
  36. data/lib/prism/dsl.rb +77 -57
  37. data/lib/prism/ffi.rb +19 -6
  38. data/lib/prism/lex_compat.rb +19 -9
  39. data/lib/prism/mutation_compiler.rb +26 -6
  40. data/lib/prism/node.rb +1314 -522
  41. data/lib/prism/node_ext.rb +102 -19
  42. data/lib/prism/parse_result.rb +58 -27
  43. data/lib/prism/ripper_compat.rb +49 -34
  44. data/lib/prism/serialize.rb +251 -227
  45. data/lib/prism/visitor.rb +15 -3
  46. data/lib/prism.rb +21 -4
  47. data/prism.gemspec +7 -9
  48. data/rbi/prism.rbi +688 -284
  49. data/rbi/prism_static.rbi +3 -0
  50. data/sig/prism.rbs +426 -156
  51. data/sig/prism_static.rbs +1 -0
  52. data/src/diagnostic.c +280 -216
  53. data/src/encoding.c +5137 -0
  54. data/src/node.c +99 -21
  55. data/src/options.c +21 -2
  56. data/src/prettyprint.c +1743 -1241
  57. data/src/prism.c +1774 -831
  58. data/src/regexp.c +15 -15
  59. data/src/serialize.c +261 -164
  60. data/src/util/pm_buffer.c +10 -1
  61. data/src/util/pm_memchr.c +1 -1
  62. data/src/util/pm_strpbrk.c +4 -4
  63. metadata +8 -10
  64. data/src/enc/pm_big5.c +0 -53
  65. data/src/enc/pm_euc_jp.c +0 -59
  66. data/src/enc/pm_gbk.c +0 -62
  67. data/src/enc/pm_shift_jis.c +0 -57
  68. data/src/enc/pm_tables.c +0 -743
  69. data/src/enc/pm_unicode.c +0 -2369
  70. data/src/enc/pm_windows_31j.c +0 -57
data/include/prism.h CHANGED
@@ -61,18 +61,6 @@ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *
61
61
  */
62
62
  PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
63
63
 
64
- /**
65
- * Register a callback that will be called when prism encounters a magic comment
66
- * with an encoding referenced that it doesn't understand. The callback should
67
- * return NULL if it also doesn't understand the encoding or it should return a
68
- * pointer to a pm_encoding_t struct that contains the functions necessary to
69
- * parse identifiers.
70
- *
71
- * @param parser The parser to register the callback with.
72
- * @param callback The callback to register.
73
- */
74
- PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_decode_callback(pm_parser_t *parser, pm_encoding_decode_callback_t callback);
75
-
76
64
  /**
77
65
  * Free any memory associated with the given parser.
78
66
  *
@@ -103,7 +91,7 @@ void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t
103
91
  * @param encoding The encoding to serialize.
104
92
  * @param buffer The buffer to serialize to.
105
93
  */
106
- void pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer);
94
+ void pm_serialize_encoding(const pm_encoding_t *encoding, pm_buffer_t *buffer);
107
95
 
108
96
  /**
109
97
  * Serialize the encoding, metadata, nodes, and constant pool.
@@ -164,6 +152,16 @@ PRISM_EXPORTED_FUNCTION void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t
164
152
  */
165
153
  PRISM_EXPORTED_FUNCTION void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
166
154
 
155
+ /**
156
+ * Parse the source and return true if it parses without errors or warnings.
157
+ *
158
+ * @param source The source to parse.
159
+ * @param size The size of the source.
160
+ * @param data The optional data to pass to the parser.
161
+ * @return True if the source parses without errors or warnings.
162
+ */
163
+ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t size, const char *data);
164
+
167
165
  /**
168
166
  * Returns a string representation of the given token type.
169
167
  *
@@ -213,7 +211,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
213
211
  * pm_node_t *root = pm_parse(&parser);
214
212
  * printf("PARSED!\n");
215
213
  *
216
- * pm_node_destroy(root);
214
+ * pm_node_destroy(&parser, root);
217
215
  * pm_parser_free(&parser);
218
216
  * }
219
217
  * ```
@@ -265,7 +263,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
265
263
  * printf("*.s%\n", (int) buffer.length, buffer.value);
266
264
  *
267
265
  * pm_buffer_free(&buffer);
268
- * pm_node_destroy(root);
266
+ * pm_node_destroy(&parser, root);
269
267
  * pm_parser_free(&parser);
270
268
  * }
271
269
  * ```
@@ -102,9 +102,15 @@ module Prism
102
102
  # Compile a CallOrWriteNode node
103
103
  alias visit_call_or_write_node visit_child_nodes
104
104
 
105
+ # Compile a CallTargetNode node
106
+ alias visit_call_target_node visit_child_nodes
107
+
105
108
  # Compile a CapturePatternNode node
106
109
  alias visit_capture_pattern_node visit_child_nodes
107
110
 
111
+ # Compile a CaseMatchNode node
112
+ alias visit_case_match_node visit_child_nodes
113
+
108
114
  # Compile a CaseNode node
109
115
  alias visit_case_node visit_child_nodes
110
116
 
@@ -240,6 +246,9 @@ module Prism
240
246
  # Compile a ImplicitNode node
241
247
  alias visit_implicit_node visit_child_nodes
242
248
 
249
+ # Compile a ImplicitRestNode node
250
+ alias visit_implicit_rest_node visit_child_nodes
251
+
243
252
  # Compile a InNode node
244
253
  alias visit_in_node visit_child_nodes
245
254
 
@@ -252,6 +261,9 @@ module Prism
252
261
  # Compile a IndexOrWriteNode node
253
262
  alias visit_index_or_write_node visit_child_nodes
254
263
 
264
+ # Compile a IndexTargetNode node
265
+ alias visit_index_target_node visit_child_nodes
266
+
255
267
  # Compile a InstanceVariableAndWriteNode node
256
268
  alias visit_instance_variable_and_write_node visit_child_nodes
257
269
 
@@ -348,6 +360,9 @@ module Prism
348
360
  # Compile a NoKeywordsParameterNode node
349
361
  alias visit_no_keywords_parameter_node visit_child_nodes
350
362
 
363
+ # Compile a NumberedParametersNode node
364
+ alias visit_numbered_parameters_node visit_child_nodes
365
+
351
366
  # Compile a NumberedReferenceReadNode node
352
367
  alias visit_numbered_reference_read_node visit_child_nodes
353
368
 
@@ -435,9 +450,6 @@ module Prism
435
450
  # Compile a StatementsNode node
436
451
  alias visit_statements_node visit_child_nodes
437
452
 
438
- # Compile a StringConcatNode node
439
- alias visit_string_concat_node visit_child_nodes
440
-
441
453
  # Compile a StringNode node
442
454
  alias visit_string_node visit_child_nodes
443
455
 
data/lib/prism/debug.rb CHANGED
@@ -103,9 +103,14 @@ module Prism
103
103
  case node
104
104
  when BlockNode, DefNode, LambdaNode
105
105
  names = node.locals
106
-
107
- params = node.parameters
108
- params = params&.parameters unless node.is_a?(DefNode)
106
+ params =
107
+ if node.is_a?(DefNode)
108
+ node.parameters
109
+ elsif node.parameters.is_a?(NumberedParametersNode)
110
+ nil
111
+ else
112
+ node.parameters&.parameters
113
+ end
109
114
 
110
115
  # prism places parameters in the same order that they appear in the
111
116
  # source. CRuby places them in the order that they need to appear
@@ -121,7 +126,7 @@ module Prism
121
126
  end
122
127
  end,
123
128
  *params.optionals.map(&:name),
124
- *((params.rest.name || :*) if params.rest && params.rest.operator != ","),
129
+ *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)),
125
130
  *params.posts.map do |post|
126
131
  if post.is_a?(RequiredParameterNode)
127
132
  post.name
@@ -133,6 +138,10 @@ module Prism
133
138
  *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
134
139
  ]
135
140
 
141
+ if params.keyword_rest.is_a?(ForwardingParameterNode)
142
+ sorted.push(:*, :&, :"...")
143
+ end
144
+
136
145
  sorted << AnonymousLocal if params.keywords.any?
137
146
 
138
147
  # Recurse down the parameter tree to find any destructured
@@ -157,15 +157,15 @@ module Prism
157
157
  *arguments,
158
158
  node.name_loc,
159
159
  CallNode.new(
160
+ 0,
160
161
  read_class.new(*arguments, node.name_loc),
161
162
  nil,
163
+ node.operator_loc.slice.chomp("="),
162
164
  node.operator_loc.copy(length: node.operator_loc.length - 1),
163
165
  nil,
164
- ArgumentsNode.new([node.value], 0, node.value.location),
166
+ ArgumentsNode.new(0, [node.value], node.value.location),
165
167
  nil,
166
168
  nil,
167
- 0,
168
- node.operator_loc.slice.chomp("="),
169
169
  node.location
170
170
  ),
171
171
  node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1),
@@ -188,6 +188,7 @@ module Prism
188
188
  IfNode.new(
189
189
  node.operator_loc,
190
190
  DefinedNode.new(nil, read_class.new(*arguments, node.name_loc), nil, node.operator_loc, node.name_loc),
191
+ node.operator_loc,
191
192
  StatementsNode.new([read_class.new(*arguments, node.name_loc)], node.location),
192
193
  ElseNode.new(
193
194
  node.operator_loc,
@@ -233,6 +233,14 @@ module Prism
233
233
  listeners[:on_call_or_write_node_leave]&.each { |listener| listener.on_call_or_write_node_leave(node) }
234
234
  end
235
235
 
236
+ # Dispatch enter and leave events for CallTargetNode nodes and continue
237
+ # walking the tree.
238
+ def visit_call_target_node(node)
239
+ listeners[:on_call_target_node_enter]&.each { |listener| listener.on_call_target_node_enter(node) }
240
+ super
241
+ listeners[:on_call_target_node_leave]&.each { |listener| listener.on_call_target_node_leave(node) }
242
+ end
243
+
236
244
  # Dispatch enter and leave events for CapturePatternNode nodes and continue
237
245
  # walking the tree.
238
246
  def visit_capture_pattern_node(node)
@@ -241,6 +249,14 @@ module Prism
241
249
  listeners[:on_capture_pattern_node_leave]&.each { |listener| listener.on_capture_pattern_node_leave(node) }
242
250
  end
243
251
 
252
+ # Dispatch enter and leave events for CaseMatchNode nodes and continue
253
+ # walking the tree.
254
+ def visit_case_match_node(node)
255
+ listeners[:on_case_match_node_enter]&.each { |listener| listener.on_case_match_node_enter(node) }
256
+ super
257
+ listeners[:on_case_match_node_leave]&.each { |listener| listener.on_case_match_node_leave(node) }
258
+ end
259
+
244
260
  # Dispatch enter and leave events for CaseNode nodes and continue
245
261
  # walking the tree.
246
262
  def visit_case_node(node)
@@ -601,6 +617,14 @@ module Prism
601
617
  listeners[:on_implicit_node_leave]&.each { |listener| listener.on_implicit_node_leave(node) }
602
618
  end
603
619
 
620
+ # Dispatch enter and leave events for ImplicitRestNode nodes and continue
621
+ # walking the tree.
622
+ def visit_implicit_rest_node(node)
623
+ listeners[:on_implicit_rest_node_enter]&.each { |listener| listener.on_implicit_rest_node_enter(node) }
624
+ super
625
+ listeners[:on_implicit_rest_node_leave]&.each { |listener| listener.on_implicit_rest_node_leave(node) }
626
+ end
627
+
604
628
  # Dispatch enter and leave events for InNode nodes and continue
605
629
  # walking the tree.
606
630
  def visit_in_node(node)
@@ -633,6 +657,14 @@ module Prism
633
657
  listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
634
658
  end
635
659
 
660
+ # Dispatch enter and leave events for IndexTargetNode nodes and continue
661
+ # walking the tree.
662
+ def visit_index_target_node(node)
663
+ listeners[:on_index_target_node_enter]&.each { |listener| listener.on_index_target_node_enter(node) }
664
+ super
665
+ listeners[:on_index_target_node_leave]&.each { |listener| listener.on_index_target_node_leave(node) }
666
+ end
667
+
636
668
  # Dispatch enter and leave events for InstanceVariableAndWriteNode nodes and continue
637
669
  # walking the tree.
638
670
  def visit_instance_variable_and_write_node(node)
@@ -889,6 +921,14 @@ module Prism
889
921
  listeners[:on_no_keywords_parameter_node_leave]&.each { |listener| listener.on_no_keywords_parameter_node_leave(node) }
890
922
  end
891
923
 
924
+ # Dispatch enter and leave events for NumberedParametersNode nodes and continue
925
+ # walking the tree.
926
+ def visit_numbered_parameters_node(node)
927
+ listeners[:on_numbered_parameters_node_enter]&.each { |listener| listener.on_numbered_parameters_node_enter(node) }
928
+ super
929
+ listeners[:on_numbered_parameters_node_leave]&.each { |listener| listener.on_numbered_parameters_node_leave(node) }
930
+ end
931
+
892
932
  # Dispatch enter and leave events for NumberedReferenceReadNode nodes and continue
893
933
  # walking the tree.
894
934
  def visit_numbered_reference_read_node(node)
@@ -1121,14 +1161,6 @@ module Prism
1121
1161
  listeners[:on_statements_node_leave]&.each { |listener| listener.on_statements_node_leave(node) }
1122
1162
  end
1123
1163
 
1124
- # Dispatch enter and leave events for StringConcatNode nodes and continue
1125
- # walking the tree.
1126
- def visit_string_concat_node(node)
1127
- listeners[:on_string_concat_node_enter]&.each { |listener| listener.on_string_concat_node_enter(node) }
1128
- super
1129
- listeners[:on_string_concat_node_leave]&.each { |listener| listener.on_string_concat_node_leave(node) }
1130
- end
1131
-
1132
1164
  # Dispatch enter and leave events for StringNode nodes and continue
1133
1165
  # walking the tree.
1134
1166
  def visit_string_node(node)
@@ -1350,12 +1382,24 @@ module Prism
1350
1382
  listeners[:on_call_or_write_node_leave]&.each { |listener| listener.on_call_or_write_node_leave(node) }
1351
1383
  end
1352
1384
 
1385
+ # Dispatch enter and leave events for CallTargetNode nodes.
1386
+ def visit_call_target_node(node)
1387
+ listeners[:on_call_target_node_enter]&.each { |listener| listener.on_call_target_node_enter(node) }
1388
+ listeners[:on_call_target_node_leave]&.each { |listener| listener.on_call_target_node_leave(node) }
1389
+ end
1390
+
1353
1391
  # Dispatch enter and leave events for CapturePatternNode nodes.
1354
1392
  def visit_capture_pattern_node(node)
1355
1393
  listeners[:on_capture_pattern_node_enter]&.each { |listener| listener.on_capture_pattern_node_enter(node) }
1356
1394
  listeners[:on_capture_pattern_node_leave]&.each { |listener| listener.on_capture_pattern_node_leave(node) }
1357
1395
  end
1358
1396
 
1397
+ # Dispatch enter and leave events for CaseMatchNode nodes.
1398
+ def visit_case_match_node(node)
1399
+ listeners[:on_case_match_node_enter]&.each { |listener| listener.on_case_match_node_enter(node) }
1400
+ listeners[:on_case_match_node_leave]&.each { |listener| listener.on_case_match_node_leave(node) }
1401
+ end
1402
+
1359
1403
  # Dispatch enter and leave events for CaseNode nodes.
1360
1404
  def visit_case_node(node)
1361
1405
  listeners[:on_case_node_enter]&.each { |listener| listener.on_case_node_enter(node) }
@@ -1626,6 +1670,12 @@ module Prism
1626
1670
  listeners[:on_implicit_node_leave]&.each { |listener| listener.on_implicit_node_leave(node) }
1627
1671
  end
1628
1672
 
1673
+ # Dispatch enter and leave events for ImplicitRestNode nodes.
1674
+ def visit_implicit_rest_node(node)
1675
+ listeners[:on_implicit_rest_node_enter]&.each { |listener| listener.on_implicit_rest_node_enter(node) }
1676
+ listeners[:on_implicit_rest_node_leave]&.each { |listener| listener.on_implicit_rest_node_leave(node) }
1677
+ end
1678
+
1629
1679
  # Dispatch enter and leave events for InNode nodes.
1630
1680
  def visit_in_node(node)
1631
1681
  listeners[:on_in_node_enter]&.each { |listener| listener.on_in_node_enter(node) }
@@ -1650,6 +1700,12 @@ module Prism
1650
1700
  listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
1651
1701
  end
1652
1702
 
1703
+ # Dispatch enter and leave events for IndexTargetNode nodes.
1704
+ def visit_index_target_node(node)
1705
+ listeners[:on_index_target_node_enter]&.each { |listener| listener.on_index_target_node_enter(node) }
1706
+ listeners[:on_index_target_node_leave]&.each { |listener| listener.on_index_target_node_leave(node) }
1707
+ end
1708
+
1653
1709
  # Dispatch enter and leave events for InstanceVariableAndWriteNode nodes.
1654
1710
  def visit_instance_variable_and_write_node(node)
1655
1711
  listeners[:on_instance_variable_and_write_node_enter]&.each { |listener| listener.on_instance_variable_and_write_node_enter(node) }
@@ -1842,6 +1898,12 @@ module Prism
1842
1898
  listeners[:on_no_keywords_parameter_node_leave]&.each { |listener| listener.on_no_keywords_parameter_node_leave(node) }
1843
1899
  end
1844
1900
 
1901
+ # Dispatch enter and leave events for NumberedParametersNode nodes.
1902
+ def visit_numbered_parameters_node(node)
1903
+ listeners[:on_numbered_parameters_node_enter]&.each { |listener| listener.on_numbered_parameters_node_enter(node) }
1904
+ listeners[:on_numbered_parameters_node_leave]&.each { |listener| listener.on_numbered_parameters_node_leave(node) }
1905
+ end
1906
+
1845
1907
  # Dispatch enter and leave events for NumberedReferenceReadNode nodes.
1846
1908
  def visit_numbered_reference_read_node(node)
1847
1909
  listeners[:on_numbered_reference_read_node_enter]&.each { |listener| listener.on_numbered_reference_read_node_enter(node) }
@@ -2016,12 +2078,6 @@ module Prism
2016
2078
  listeners[:on_statements_node_leave]&.each { |listener| listener.on_statements_node_leave(node) }
2017
2079
  end
2018
2080
 
2019
- # Dispatch enter and leave events for StringConcatNode nodes.
2020
- def visit_string_concat_node(node)
2021
- listeners[:on_string_concat_node_enter]&.each { |listener| listener.on_string_concat_node_enter(node) }
2022
- listeners[:on_string_concat_node_leave]&.each { |listener| listener.on_string_concat_node_leave(node) }
2023
- end
2024
-
2025
2081
  # Dispatch enter and leave events for StringNode nodes.
2026
2082
  def visit_string_node(node)
2027
2083
  listeners[:on_string_node_enter]&.each { |listener| listener.on_string_node_enter(node) }