prism 0.24.0 → 0.29.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/BSDmakefile +58 -0
  3. data/CHANGELOG.md +132 -1
  4. data/Makefile +25 -18
  5. data/README.md +45 -6
  6. data/config.yml +828 -25
  7. data/docs/build_system.md +31 -0
  8. data/docs/configuration.md +4 -0
  9. data/docs/cruby_compilation.md +1 -1
  10. data/docs/parser_translation.md +14 -9
  11. data/docs/releasing.md +7 -9
  12. data/docs/ripper_translation.md +50 -0
  13. data/docs/ruby_api.md +1 -0
  14. data/docs/serialization.md +26 -5
  15. data/ext/prism/api_node.c +1037 -936
  16. data/ext/prism/api_pack.c +9 -0
  17. data/ext/prism/extconf.rb +62 -18
  18. data/ext/prism/extension.c +351 -71
  19. data/ext/prism/extension.h +5 -4
  20. data/include/prism/ast.h +539 -101
  21. data/include/prism/defines.h +106 -2
  22. data/include/prism/diagnostic.h +168 -74
  23. data/include/prism/encoding.h +22 -4
  24. data/include/prism/node.h +93 -0
  25. data/include/prism/options.h +84 -9
  26. data/include/prism/pack.h +11 -0
  27. data/include/prism/parser.h +213 -54
  28. data/include/prism/prettyprint.h +8 -0
  29. data/include/prism/static_literals.h +120 -0
  30. data/include/prism/util/pm_buffer.h +65 -2
  31. data/include/prism/util/pm_constant_pool.h +18 -1
  32. data/include/prism/util/pm_integer.h +119 -0
  33. data/include/prism/util/pm_list.h +1 -1
  34. data/include/prism/util/pm_newline_list.h +8 -0
  35. data/include/prism/util/pm_string.h +26 -2
  36. data/include/prism/version.h +2 -2
  37. data/include/prism.h +59 -1
  38. data/lib/prism/compiler.rb +8 -1
  39. data/lib/prism/debug.rb +46 -3
  40. data/lib/prism/desugar_compiler.rb +5 -3
  41. data/lib/prism/dispatcher.rb +29 -0
  42. data/lib/prism/dot_visitor.rb +141 -54
  43. data/lib/prism/dsl.rb +48 -36
  44. data/lib/prism/ffi.rb +82 -17
  45. data/lib/prism/inspect_visitor.rb +2156 -0
  46. data/lib/prism/lex_compat.rb +34 -15
  47. data/lib/prism/mutation_compiler.rb +13 -2
  48. data/lib/prism/node.rb +4453 -4459
  49. data/lib/prism/node_ext.rb +249 -30
  50. data/lib/prism/pack.rb +4 -0
  51. data/lib/prism/parse_result/comments.rb +35 -18
  52. data/lib/prism/parse_result/newlines.rb +2 -2
  53. data/lib/prism/parse_result.rb +218 -43
  54. data/lib/prism/pattern.rb +28 -10
  55. data/lib/prism/polyfill/byteindex.rb +13 -0
  56. data/lib/prism/polyfill/unpack1.rb +14 -0
  57. data/lib/prism/reflection.rb +411 -0
  58. data/lib/prism/serialize.rb +480 -112
  59. data/lib/prism/translation/parser/compiler.rb +376 -88
  60. data/lib/prism/translation/parser/lexer.rb +103 -22
  61. data/lib/prism/translation/parser/rubocop.rb +41 -13
  62. data/lib/prism/translation/parser.rb +123 -11
  63. data/lib/prism/translation/parser33.rb +1 -1
  64. data/lib/prism/translation/parser34.rb +1 -1
  65. data/lib/prism/translation/ripper/sexp.rb +125 -0
  66. data/lib/prism/translation/ripper/shim.rb +5 -0
  67. data/lib/prism/translation/ripper.rb +3216 -462
  68. data/lib/prism/translation/ruby_parser.rb +111 -56
  69. data/lib/prism/translation.rb +3 -1
  70. data/lib/prism/visitor.rb +10 -0
  71. data/lib/prism.rb +12 -20
  72. data/prism.gemspec +46 -14
  73. data/rbi/prism/compiler.rbi +12 -0
  74. data/rbi/prism/inspect_visitor.rbi +12 -0
  75. data/rbi/prism/node.rbi +8712 -0
  76. data/rbi/prism/node_ext.rbi +107 -0
  77. data/rbi/prism/parse_result.rbi +358 -0
  78. data/rbi/prism/reflection.rbi +58 -0
  79. data/rbi/prism/translation/parser.rbi +11 -0
  80. data/rbi/prism/translation/parser33.rbi +6 -0
  81. data/rbi/prism/translation/parser34.rbi +6 -0
  82. data/rbi/prism/translation/ripper.rbi +15 -0
  83. data/rbi/prism/visitor.rbi +470 -0
  84. data/rbi/prism.rbi +38 -7748
  85. data/sig/prism/compiler.rbs +9 -0
  86. data/sig/prism/dispatcher.rbs +16 -0
  87. data/sig/prism/dot_visitor.rbs +6 -0
  88. data/sig/prism/dsl.rbs +462 -0
  89. data/sig/prism/inspect_visitor.rbs +22 -0
  90. data/sig/prism/lex_compat.rbs +10 -0
  91. data/sig/prism/mutation_compiler.rbs +158 -0
  92. data/sig/prism/node.rbs +3558 -0
  93. data/sig/prism/node_ext.rbs +82 -0
  94. data/sig/prism/pack.rbs +43 -0
  95. data/sig/prism/parse_result.rbs +160 -0
  96. data/sig/prism/pattern.rbs +13 -0
  97. data/sig/prism/reflection.rbs +50 -0
  98. data/sig/prism/serialize.rbs +6 -0
  99. data/sig/prism/visitor.rbs +168 -0
  100. data/sig/prism.rbs +188 -4767
  101. data/src/diagnostic.c +636 -230
  102. data/src/encoding.c +211 -108
  103. data/src/node.c +7555 -451
  104. data/src/options.c +66 -31
  105. data/src/pack.c +33 -17
  106. data/src/prettyprint.c +1383 -1431
  107. data/src/prism.c +4734 -1310
  108. data/src/regexp.c +17 -2
  109. data/src/serialize.c +68 -46
  110. data/src/static_literals.c +638 -0
  111. data/src/token_type.c +10 -9
  112. data/src/util/pm_buffer.c +147 -20
  113. data/src/util/pm_char.c +4 -4
  114. data/src/util/pm_constant_pool.c +35 -11
  115. data/src/util/pm_integer.c +642 -0
  116. data/src/util/pm_list.c +1 -1
  117. data/src/util/pm_newline_list.c +14 -5
  118. data/src/util/pm_string.c +134 -5
  119. data/src/util/pm_string_list.c +2 -2
  120. metadata +41 -9
  121. data/docs/ripper.md +0 -36
  122. data/include/prism/util/pm_state_stack.h +0 -42
  123. data/lib/prism/node_inspector.rb +0 -68
  124. data/rbi/prism_static.rbi +0 -207
  125. data/sig/prism_static.rbs +0 -201
  126. data/src/util/pm_state_stack.c +0 -25
@@ -10,6 +10,23 @@ module Prism
10
10
  # generally lines up. However, there are a few cases that require special
11
11
  # handling.
12
12
  class LexCompat # :nodoc:
13
+ # A result class specialized for holding tokens produced by the lexer.
14
+ class Result < Prism::Result
15
+ # The list of tokens that were produced by the lexer.
16
+ attr_reader :value
17
+
18
+ # Create a new lex compat result object with the given values.
19
+ def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
20
+ @value = value
21
+ super(comments, magic_comments, data_loc, errors, warnings, source)
22
+ end
23
+
24
+ # Implement the hash pattern matching interface for Result.
25
+ def deconstruct_keys(keys)
26
+ super.merge!(value: value)
27
+ end
28
+ end
29
+
13
30
  # This is a mapping of prism token types to Ripper token types. This is a
14
31
  # many-to-one mapping because we split up our token types, whereas Ripper
15
32
  # tends to group them.
@@ -185,6 +202,8 @@ module Prism
185
202
  # However, we add a couple of convenience methods onto them to make them a
186
203
  # little easier to work with. We delegate all other methods to the array.
187
204
  class Token < SimpleDelegator
205
+ # @dynamic initialize, each, []
206
+
188
207
  # The location of the token in the source.
189
208
  def location
190
209
  self[0]
@@ -241,10 +260,10 @@ module Prism
241
260
  def ==(other) # :nodoc:
242
261
  return false unless self[0...-1] == other[0...-1]
243
262
 
244
- if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
245
- other[4] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED > 0
263
+ if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
264
+ other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0
246
265
  else
247
- self[4] == other[4]
266
+ self[3] == other[3]
248
267
  end
249
268
  end
250
269
  end
@@ -308,7 +327,7 @@ module Prism
308
327
  def to_a
309
328
  embexpr_balance = 0
310
329
 
311
- tokens.each_with_object([]) do |token, results|
330
+ tokens.each_with_object([]) do |token, results| #$ Array[Token]
312
331
  case token.event
313
332
  when :on_embexpr_beg
314
333
  embexpr_balance += 1
@@ -409,7 +428,7 @@ module Prism
409
428
  # If every line in the heredoc is blank, we still need to split up the
410
429
  # string content token into multiple tokens.
411
430
  if dedent.nil?
412
- results = []
431
+ results = [] #: Array[Token]
413
432
  embexpr_balance = 0
414
433
 
415
434
  tokens.each do |token|
@@ -444,7 +463,7 @@ module Prism
444
463
  # If the minimum common whitespace is 0, then we need to concatenate
445
464
  # string nodes together that are immediately adjacent.
446
465
  if dedent == 0
447
- results = []
466
+ results = [] #: Array[Token]
448
467
  embexpr_balance = 0
449
468
 
450
469
  index = 0
@@ -477,7 +496,7 @@ module Prism
477
496
  # insert on_ignored_sp tokens for the amount of dedent that we need to
478
497
  # perform. We also need to remove the dedent from the beginning of
479
498
  # each line of plain string content tokens.
480
- results = []
499
+ results = [] #: Array[Token]
481
500
  dedent_next = true
482
501
  embexpr_balance = 0
483
502
 
@@ -526,7 +545,7 @@ module Prism
526
545
  # dedent from the beginning of the line.
527
546
  if (dedent > 0) && (dedent_next || index > 0)
528
547
  deleting = 0
529
- deleted_chars = []
548
+ deleted_chars = [] #: Array[String]
530
549
 
531
550
  # Gather up all of the characters that we're going to
532
551
  # delete, stopping when you hit a character that would put
@@ -603,15 +622,15 @@ module Prism
603
622
  end
604
623
 
605
624
  def result
606
- tokens = []
625
+ tokens = [] #: Array[LexCompat::Token]
607
626
 
608
627
  state = :default
609
- heredoc_stack = [[]]
628
+ heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]]
610
629
 
611
630
  result = Prism.lex(source, **options)
612
631
  result_value = result.value
613
- previous_state = nil
614
- last_heredoc_end = nil
632
+ previous_state = nil #: Ripper::Lexer::State?
633
+ last_heredoc_end = nil #: Integer?
615
634
 
616
635
  # In previous versions of Ruby, Ripper wouldn't flush the bom before the
617
636
  # first token, so we had to have a hack in place to account for that. This
@@ -842,7 +861,7 @@ module Prism
842
861
  # We sort by location to compare against Ripper's output
843
862
  tokens.sort_by!(&:location)
844
863
 
845
- ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, [])
864
+ Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source))
846
865
  end
847
866
  end
848
867
 
@@ -858,8 +877,8 @@ module Prism
858
877
  end
859
878
 
860
879
  def result
861
- previous = []
862
- results = []
880
+ previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
881
+ results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
863
882
 
864
883
  lex(source).each do |token|
865
884
  case token[1]
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  =begin
3
4
  This file is generated by the templates/template.rb script and should not be
4
5
  modified manually. See templates/lib/prism/mutation_compiler.rb.erb
@@ -192,7 +193,7 @@ module Prism
192
193
 
193
194
  # Copy a ConstantPathNode node
194
195
  def visit_constant_path_node(node)
195
- node.copy(parent: visit(node.parent), child: visit(node.child))
196
+ node.copy(parent: visit(node.parent))
196
197
  end
197
198
 
198
199
  # Copy a ConstantPathOperatorWriteNode node
@@ -207,7 +208,7 @@ module Prism
207
208
 
208
209
  # Copy a ConstantPathTargetNode node
209
210
  def visit_constant_path_target_node(node)
210
- node.copy(parent: visit(node.parent), child: visit(node.child))
211
+ node.copy(parent: visit(node.parent))
211
212
  end
212
213
 
213
214
  # Copy a ConstantPathWriteNode node
@@ -445,6 +446,11 @@ module Prism
445
446
  node.copy(parts: visit_all(node.parts))
446
447
  end
447
448
 
449
+ # Copy a ItParametersNode node
450
+ def visit_it_parameters_node(node)
451
+ node.copy
452
+ end
453
+
448
454
  # Copy a KeywordHashNode node
449
455
  def visit_keyword_hash_node(node)
450
456
  node.copy(elements: visit_all(node.elements))
@@ -665,6 +671,11 @@ module Prism
665
671
  node.copy
666
672
  end
667
673
 
674
+ # Copy a ShareableConstantNode node
675
+ def visit_shareable_constant_node(node)
676
+ node.copy(write: visit(node.write))
677
+ end
678
+
668
679
  # Copy a SingletonClassNode node
669
680
  def visit_singleton_class_node(node)
670
681
  node.copy(expression: visit(node.expression), body: visit(node.body))