prism 0.30.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +31 -1
  3. data/README.md +3 -1
  4. data/config.yml +185 -126
  5. data/docs/serialization.md +3 -0
  6. data/ext/prism/api_node.c +2843 -2085
  7. data/ext/prism/extconf.rb +1 -1
  8. data/ext/prism/extension.c +35 -25
  9. data/ext/prism/extension.h +2 -2
  10. data/include/prism/ast.h +1048 -69
  11. data/include/prism/defines.h +9 -0
  12. data/include/prism/diagnostic.h +11 -3
  13. data/include/prism/options.h +55 -1
  14. data/include/prism/parser.h +27 -3
  15. data/include/prism/regexp.h +2 -1
  16. data/include/prism/util/pm_integer.h +6 -6
  17. data/include/prism/util/pm_newline_list.h +11 -0
  18. data/include/prism/util/pm_string.h +1 -0
  19. data/include/prism/version.h +3 -3
  20. data/lib/prism/desugar_compiler.rb +111 -74
  21. data/lib/prism/dispatcher.rb +2 -1
  22. data/lib/prism/dot_visitor.rb +21 -31
  23. data/lib/prism/dsl.rb +656 -471
  24. data/lib/prism/ffi.rb +3 -0
  25. data/lib/prism/inspect_visitor.rb +285 -57
  26. data/lib/prism/mutation_compiler.rb +5 -5
  27. data/lib/prism/node.rb +2282 -4754
  28. data/lib/prism/node_ext.rb +72 -11
  29. data/lib/prism/parse_result/errors.rb +65 -0
  30. data/lib/prism/parse_result/newlines.rb +28 -28
  31. data/lib/prism/parse_result.rb +25 -2
  32. data/lib/prism/reflection.rb +7 -7
  33. data/lib/prism/serialize.rb +468 -610
  34. data/lib/prism/translation/parser/compiler.rb +18 -18
  35. data/lib/prism/translation/parser/lexer.rb +1 -1
  36. data/lib/prism/translation/parser.rb +3 -3
  37. data/lib/prism/translation/ripper.rb +14 -14
  38. data/lib/prism/translation/ruby_parser.rb +43 -7
  39. data/prism.gemspec +3 -1
  40. data/rbi/prism/dsl.rbi +521 -0
  41. data/rbi/prism/node.rbi +1456 -5616
  42. data/rbi/prism.rbi +16 -16
  43. data/sig/prism/dsl.rbs +189 -305
  44. data/sig/prism/node.rbs +702 -603
  45. data/sig/prism/parse_result.rbs +2 -0
  46. data/src/diagnostic.c +22 -6
  47. data/src/node.c +277 -284
  48. data/src/options.c +18 -0
  49. data/src/prettyprint.c +99 -108
  50. data/src/prism.c +1282 -760
  51. data/src/regexp.c +72 -4
  52. data/src/serialize.c +165 -50
  53. data/src/token_type.c +2 -2
  54. data/src/util/pm_integer.c +14 -14
  55. data/src/util/pm_newline_list.c +29 -0
  56. data/src/util/pm_string.c +9 -5
  57. metadata +4 -2
@@ -181,7 +181,7 @@ module Prism
181
181
  if (rescue_clause = node.rescue_clause)
182
182
  begin
183
183
  find_start_offset = (rescue_clause.reference&.location || rescue_clause.exceptions.last&.location || rescue_clause.keyword_loc).end_offset
184
- find_end_offset = (rescue_clause.statements&.location&.start_offset || rescue_clause.consequent&.location&.start_offset || (find_start_offset + 1))
184
+ find_end_offset = (rescue_clause.statements&.location&.start_offset || rescue_clause.subsequent&.location&.start_offset || (find_start_offset + 1))
185
185
 
186
186
  rescue_bodies << builder.rescue_body(
187
187
  token(rescue_clause.keyword_loc),
@@ -191,7 +191,7 @@ module Prism
191
191
  srange_find(find_start_offset, find_end_offset, [";"]),
192
192
  visit(rescue_clause.statements)
193
193
  )
194
- end until (rescue_clause = rescue_clause.consequent).nil?
194
+ end until (rescue_clause = rescue_clause.subsequent).nil?
195
195
  end
196
196
 
197
197
  begin_body =
@@ -410,8 +410,8 @@ module Prism
410
410
  token(node.case_keyword_loc),
411
411
  visit(node.predicate),
412
412
  visit_all(node.conditions),
413
- token(node.consequent&.else_keyword_loc),
414
- visit(node.consequent),
413
+ token(node.else_clause&.else_keyword_loc),
414
+ visit(node.else_clause),
415
415
  token(node.end_keyword_loc)
416
416
  )
417
417
  end
@@ -423,8 +423,8 @@ module Prism
423
423
  token(node.case_keyword_loc),
424
424
  visit(node.predicate),
425
425
  visit_all(node.conditions),
426
- token(node.consequent&.else_keyword_loc),
427
- visit(node.consequent),
426
+ token(node.else_clause&.else_keyword_loc),
427
+ visit(node.else_clause),
428
428
  token(node.end_keyword_loc)
429
429
  )
430
430
  end
@@ -858,8 +858,8 @@ module Prism
858
858
  visit(node.predicate),
859
859
  token(node.then_keyword_loc),
860
860
  visit(node.statements),
861
- token(node.consequent.else_keyword_loc),
862
- visit(node.consequent)
861
+ token(node.subsequent.else_keyword_loc),
862
+ visit(node.subsequent)
863
863
  )
864
864
  elsif node.if_keyword_loc.start_offset == node.location.start_offset
865
865
  builder.condition(
@@ -868,16 +868,16 @@ module Prism
868
868
  if node.then_keyword_loc
869
869
  token(node.then_keyword_loc)
870
870
  else
871
- srange_find(node.predicate.location.end_offset, (node.statements&.location || node.consequent&.location || node.end_keyword_loc).start_offset, [";"])
871
+ srange_find(node.predicate.location.end_offset, (node.statements&.location || node.subsequent&.location || node.end_keyword_loc).start_offset, [";"])
872
872
  end,
873
873
  visit(node.statements),
874
- case node.consequent
874
+ case node.subsequent
875
875
  when IfNode
876
- token(node.consequent.if_keyword_loc)
876
+ token(node.subsequent.if_keyword_loc)
877
877
  when ElseNode
878
- token(node.consequent.else_keyword_loc)
878
+ token(node.subsequent.else_keyword_loc)
879
879
  end,
880
- visit(node.consequent),
880
+ visit(node.subsequent),
881
881
  if node.if_keyword != "elsif"
882
882
  token(node.end_keyword_loc)
883
883
  end
@@ -885,7 +885,7 @@ module Prism
885
885
  else
886
886
  builder.condition_mod(
887
887
  visit(node.statements),
888
- visit(node.consequent),
888
+ visit(node.subsequent),
889
889
  token(node.if_keyword_loc),
890
890
  visit(node.predicate)
891
891
  )
@@ -1784,16 +1784,16 @@ module Prism
1784
1784
  if node.then_keyword_loc
1785
1785
  token(node.then_keyword_loc)
1786
1786
  else
1787
- srange_find(node.predicate.location.end_offset, (node.statements&.location || node.consequent&.location || node.end_keyword_loc).start_offset, [";"])
1787
+ srange_find(node.predicate.location.end_offset, (node.statements&.location || node.else_clause&.location || node.end_keyword_loc).start_offset, [";"])
1788
1788
  end,
1789
- visit(node.consequent),
1790
- token(node.consequent&.else_keyword_loc),
1789
+ visit(node.else_clause),
1790
+ token(node.else_clause&.else_keyword_loc),
1791
1791
  visit(node.statements),
1792
1792
  token(node.end_keyword_loc)
1793
1793
  )
1794
1794
  else
1795
1795
  builder.condition_mod(
1796
- visit(node.consequent),
1796
+ visit(node.else_clause),
1797
1797
  visit(node.statements),
1798
1798
  token(node.keyword_loc),
1799
1799
  visit(node.predicate)
@@ -339,7 +339,7 @@ module Prism
339
339
  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
340
340
  end
341
341
  when :tSYMBEG
342
- if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
342
+ if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
343
343
  next_location = token.location.join(next_token.location)
344
344
  type = :tSYMBOL
345
345
  value = next_token.value
@@ -51,7 +51,7 @@ module Prism
51
51
  source = source_buffer.source
52
52
 
53
53
  offset_cache = build_offset_cache(source)
54
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
54
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
55
55
 
56
56
  build_ast(result.value, offset_cache)
57
57
  ensure
@@ -64,7 +64,7 @@ module Prism
64
64
  source = source_buffer.source
65
65
 
66
66
  offset_cache = build_offset_cache(source)
67
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
67
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
68
68
 
69
69
  [
70
70
  build_ast(result.value, offset_cache),
@@ -83,7 +83,7 @@ module Prism
83
83
  offset_cache = build_offset_cache(source)
84
84
  result =
85
85
  begin
86
- unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
86
+ unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
87
87
  rescue ::Parser::SyntaxError
88
88
  raise if !recover
89
89
  end
@@ -1273,8 +1273,8 @@ module Prism
1273
1273
  def visit_case_node(node)
1274
1274
  predicate = visit(node.predicate)
1275
1275
  clauses =
1276
- node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
1277
- on_when(*visit(condition), consequent)
1276
+ node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
1277
+ on_when(*visit(condition), current)
1278
1278
  end
1279
1279
 
1280
1280
  bounds(node.location)
@@ -1286,8 +1286,8 @@ module Prism
1286
1286
  def visit_case_match_node(node)
1287
1287
  predicate = visit(node.predicate)
1288
1288
  clauses =
1289
- node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
1290
- on_in(*visit(condition), consequent)
1289
+ node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
1290
+ on_in(*visit(condition), current)
1291
1291
  end
1292
1292
 
1293
1293
  bounds(node.location)
@@ -1908,7 +1908,7 @@ module Prism
1908
1908
  if node.then_keyword == "?"
1909
1909
  predicate = visit(node.predicate)
1910
1910
  truthy = visit(node.statements.body.first)
1911
- falsy = visit(node.consequent.statements.body.first)
1911
+ falsy = visit(node.subsequent.statements.body.first)
1912
1912
 
1913
1913
  bounds(node.location)
1914
1914
  on_ifop(predicate, truthy, falsy)
@@ -1921,13 +1921,13 @@ module Prism
1921
1921
  else
1922
1922
  visit(node.statements)
1923
1923
  end
1924
- consequent = visit(node.consequent)
1924
+ subsequent = visit(node.subsequent)
1925
1925
 
1926
1926
  bounds(node.location)
1927
1927
  if node.if_keyword == "if"
1928
- on_if(predicate, statements, consequent)
1928
+ on_if(predicate, statements, subsequent)
1929
1929
  else
1930
- on_elsif(predicate, statements, consequent)
1930
+ on_elsif(predicate, statements, subsequent)
1931
1931
  end
1932
1932
  else
1933
1933
  statements = visit(node.statements.body.first)
@@ -1960,7 +1960,7 @@ module Prism
1960
1960
  # ^^^^^^^^^^^^^^^^^^^^^
1961
1961
  def visit_in_node(node)
1962
1962
  # This is a special case where we're not going to call on_in directly
1963
- # because we don't have access to the consequent. Instead, we'll return
1963
+ # because we don't have access to the subsequent. Instead, we'll return
1964
1964
  # the component parts and let the parent node handle it.
1965
1965
  pattern = visit_pattern_node(node.pattern)
1966
1966
  statements =
@@ -2808,10 +2808,10 @@ module Prism
2808
2808
  visit(node.statements)
2809
2809
  end
2810
2810
 
2811
- consequent = visit(node.consequent)
2811
+ subsequent = visit(node.subsequent)
2812
2812
 
2813
2813
  bounds(node.location)
2814
- on_rescue(exceptions, reference, statements, consequent)
2814
+ on_rescue(exceptions, reference, statements, subsequent)
2815
2815
  end
2816
2816
 
2817
2817
  # def foo(*bar); end
@@ -3132,10 +3132,10 @@ module Prism
3132
3132
  else
3133
3133
  visit(node.statements)
3134
3134
  end
3135
- consequent = visit(node.consequent)
3135
+ else_clause = visit(node.else_clause)
3136
3136
 
3137
3137
  bounds(node.location)
3138
- on_unless(predicate, statements, consequent)
3138
+ on_unless(predicate, statements, else_clause)
3139
3139
  else
3140
3140
  statements = visit(node.statements.body.first)
3141
3141
  predicate = visit(node.predicate)
@@ -3176,7 +3176,7 @@ module Prism
3176
3176
  # ^^^^^^^^^^^^^
3177
3177
  def visit_when_node(node)
3178
3178
  # This is a special case where we're not going to call on_when directly
3179
- # because we don't have access to the consequent. Instead, we'll return
3179
+ # because we don't have access to the subsequent. Instead, we'll return
3180
3180
  # the component parts and let the parent node handle it.
3181
3181
  conditions = visit_arguments(node.conditions)
3182
3182
  statements =
@@ -55,7 +55,19 @@ module Prism
55
55
  # a and b
56
56
  # ^^^^^^^
57
57
  def visit_and_node(node)
58
- s(node, :and, visit(node.left), visit(node.right))
58
+ left = visit(node.left)
59
+
60
+ if left[0] == :and
61
+ # ruby_parser has the and keyword as right-associative as opposed to
62
+ # prism which has it as left-associative. We reverse that
63
+ # associativity here.
64
+ nest = left
65
+ nest = nest[2] while nest[2][0] == :and
66
+ nest[2] = s(node, :and, nest[2], visit(node.right))
67
+ left
68
+ else
69
+ s(node, :and, left, visit(node.right))
70
+ end
59
71
  end
60
72
 
61
73
  # []
@@ -135,7 +147,7 @@ module Prism
135
147
  end
136
148
 
137
149
  current = node.rescue_clause
138
- until (current = current.consequent).nil?
150
+ until (current = current.subsequent).nil?
139
151
  result << visit(current)
140
152
  end
141
153
  end
@@ -251,6 +263,11 @@ module Prism
251
263
  when RegularExpressionNode, InterpolatedRegularExpressionNode
252
264
  return s(node, :match2, visit(node.receiver), visit(node.arguments.arguments.first))
253
265
  end
266
+
267
+ case node.arguments.arguments.first
268
+ when RegularExpressionNode, InterpolatedRegularExpressionNode
269
+ return s(node, :match3, visit(node.arguments.arguments.first), visit(node.receiver))
270
+ end
254
271
  end
255
272
  end
256
273
 
@@ -330,13 +347,13 @@ module Prism
330
347
  # case foo; when bar; end
331
348
  # ^^^^^^^^^^^^^^^^^^^^^^^
332
349
  def visit_case_node(node)
333
- s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
350
+ s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
334
351
  end
335
352
 
336
353
  # case foo; in bar; end
337
354
  # ^^^^^^^^^^^^^^^^^^^^^
338
355
  def visit_case_match_node(node)
339
- s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
356
+ s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
340
357
  end
341
358
 
342
359
  # class Foo; end
@@ -683,7 +700,7 @@ module Prism
683
700
  # foo ? bar : baz
684
701
  # ^^^^^^^^^^^^^^^
685
702
  def visit_if_node(node)
686
- s(node, :if, visit(node.predicate), visit(node.statements), visit(node.consequent))
703
+ s(node, :if, visit(node.predicate), visit(node.statements), visit(node.subsequent))
687
704
  end
688
705
 
689
706
  # 1i
@@ -876,6 +893,13 @@ module Prism
876
893
  visited << result
877
894
  end
878
895
  elsif result[0] == :dstr
896
+ if !visited.empty? && part.parts[0].is_a?(StringNode)
897
+ # If we are in the middle of an implicitly concatenated string,
898
+ # we should not have a bare string as the first part. In this
899
+ # case we need to visit just that first part and then we can
900
+ # push the rest of the parts onto the visited array.
901
+ result[1] = visit(part.parts[0])
902
+ end
879
903
  visited.concat(result[1..-1])
880
904
  else
881
905
  visited << result
@@ -1136,7 +1160,19 @@ module Prism
1136
1160
  # a or b
1137
1161
  # ^^^^^^
1138
1162
  def visit_or_node(node)
1139
- s(node, :or, visit(node.left), visit(node.right))
1163
+ left = visit(node.left)
1164
+
1165
+ if left[0] == :or
1166
+ # ruby_parser has the or keyword as right-associative as opposed to
1167
+ # prism which has it as left-associative. We reverse that
1168
+ # associativity here.
1169
+ nest = left
1170
+ nest = nest[2] while nest[2][0] == :or
1171
+ nest[2] = s(node, :or, nest[2], visit(node.right))
1172
+ left
1173
+ else
1174
+ s(node, :or, left, visit(node.right))
1175
+ end
1140
1176
  end
1141
1177
 
1142
1178
  # def foo(bar, *baz); end
@@ -1434,7 +1470,7 @@ module Prism
1434
1470
  # bar unless foo
1435
1471
  # ^^^^^^^^^^^^^^
1436
1472
  def visit_unless_node(node)
1437
- s(node, :if, visit(node.predicate), visit(node.consequent), visit(node.statements))
1473
+ s(node, :if, visit(node.predicate), visit(node.else_clause), visit(node.statements))
1438
1474
  end
1439
1475
 
1440
1476
  # until foo; bar end
data/prism.gemspec CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |spec|
4
4
  spec.name = "prism"
5
- spec.version = "0.30.0"
5
+ spec.version = "1.0.0"
6
6
  spec.authors = ["Shopify"]
7
7
  spec.email = ["ruby@shopify.com"]
8
8
 
@@ -82,6 +82,7 @@ Gem::Specification.new do |spec|
82
82
  "lib/prism/pack.rb",
83
83
  "lib/prism/parse_result.rb",
84
84
  "lib/prism/parse_result/comments.rb",
85
+ "lib/prism/parse_result/errors.rb",
85
86
  "lib/prism/parse_result/newlines.rb",
86
87
  "lib/prism/pattern.rb",
87
88
  "lib/prism/polyfill/byteindex.rb",
@@ -103,6 +104,7 @@ Gem::Specification.new do |spec|
103
104
  "prism.gemspec",
104
105
  "rbi/prism.rbi",
105
106
  "rbi/prism/compiler.rbi",
107
+ "rbi/prism/dsl.rbi",
106
108
  "rbi/prism/inspect_visitor.rbi",
107
109
  "rbi/prism/node_ext.rbi",
108
110
  "rbi/prism/node.rbi",