prism 0.29.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/Makefile +1 -1
  5. data/README.md +4 -0
  6. data/config.yml +920 -148
  7. data/docs/build_system.md +8 -11
  8. data/docs/fuzzing.md +1 -1
  9. data/docs/parsing_rules.md +4 -1
  10. data/docs/relocation.md +34 -0
  11. data/docs/ripper_translation.md +22 -0
  12. data/docs/serialization.md +3 -0
  13. data/ext/prism/api_node.c +2863 -2079
  14. data/ext/prism/extconf.rb +14 -37
  15. data/ext/prism/extension.c +241 -391
  16. data/ext/prism/extension.h +2 -2
  17. data/include/prism/ast.h +2156 -453
  18. data/include/prism/defines.h +58 -7
  19. data/include/prism/diagnostic.h +24 -6
  20. data/include/prism/node.h +0 -21
  21. data/include/prism/options.h +94 -3
  22. data/include/prism/parser.h +82 -40
  23. data/include/prism/regexp.h +18 -8
  24. data/include/prism/static_literals.h +3 -2
  25. data/include/prism/util/pm_char.h +1 -2
  26. data/include/prism/util/pm_constant_pool.h +0 -8
  27. data/include/prism/util/pm_integer.h +22 -15
  28. data/include/prism/util/pm_newline_list.h +11 -0
  29. data/include/prism/util/pm_string.h +28 -12
  30. data/include/prism/version.h +3 -3
  31. data/include/prism.h +47 -11
  32. data/lib/prism/compiler.rb +3 -0
  33. data/lib/prism/desugar_compiler.rb +111 -74
  34. data/lib/prism/dispatcher.rb +16 -1
  35. data/lib/prism/dot_visitor.rb +55 -34
  36. data/lib/prism/dsl.rb +660 -468
  37. data/lib/prism/ffi.rb +113 -8
  38. data/lib/prism/inspect_visitor.rb +296 -64
  39. data/lib/prism/lex_compat.rb +1 -1
  40. data/lib/prism/mutation_compiler.rb +11 -6
  41. data/lib/prism/node.rb +4262 -5023
  42. data/lib/prism/node_ext.rb +91 -14
  43. data/lib/prism/parse_result/comments.rb +0 -7
  44. data/lib/prism/parse_result/errors.rb +65 -0
  45. data/lib/prism/parse_result/newlines.rb +101 -11
  46. data/lib/prism/parse_result.rb +183 -6
  47. data/lib/prism/reflection.rb +12 -10
  48. data/lib/prism/relocation.rb +504 -0
  49. data/lib/prism/serialize.rb +496 -609
  50. data/lib/prism/string_query.rb +30 -0
  51. data/lib/prism/translation/parser/compiler.rb +185 -155
  52. data/lib/prism/translation/parser/lexer.rb +26 -4
  53. data/lib/prism/translation/parser.rb +9 -4
  54. data/lib/prism/translation/ripper.rb +23 -25
  55. data/lib/prism/translation/ruby_parser.rb +86 -17
  56. data/lib/prism/visitor.rb +3 -0
  57. data/lib/prism.rb +6 -8
  58. data/prism.gemspec +9 -5
  59. data/rbi/prism/dsl.rbi +521 -0
  60. data/rbi/prism/node.rbi +1115 -1120
  61. data/rbi/prism/parse_result.rbi +29 -0
  62. data/rbi/prism/string_query.rbi +12 -0
  63. data/rbi/prism/visitor.rbi +3 -0
  64. data/rbi/prism.rbi +36 -30
  65. data/sig/prism/dsl.rbs +190 -303
  66. data/sig/prism/mutation_compiler.rbs +1 -0
  67. data/sig/prism/node.rbs +678 -632
  68. data/sig/prism/parse_result.rbs +22 -0
  69. data/sig/prism/relocation.rbs +185 -0
  70. data/sig/prism/string_query.rbs +11 -0
  71. data/sig/prism/visitor.rbs +1 -0
  72. data/sig/prism.rbs +103 -64
  73. data/src/diagnostic.c +64 -28
  74. data/src/node.c +502 -1739
  75. data/src/options.c +76 -27
  76. data/src/prettyprint.c +188 -112
  77. data/src/prism.c +3376 -2293
  78. data/src/regexp.c +208 -71
  79. data/src/serialize.c +182 -50
  80. data/src/static_literals.c +64 -85
  81. data/src/token_type.c +4 -4
  82. data/src/util/pm_char.c +1 -1
  83. data/src/util/pm_constant_pool.c +0 -8
  84. data/src/util/pm_integer.c +53 -25
  85. data/src/util/pm_newline_list.c +29 -0
  86. data/src/util/pm_string.c +131 -80
  87. data/src/util/pm_strpbrk.c +32 -6
  88. metadata +11 -7
  89. data/include/prism/util/pm_string_list.h +0 -44
  90. data/lib/prism/debug.rb +0 -249
  91. data/lib/prism/translation/parser/rubocop.rb +0 -73
  92. data/src/util/pm_string_list.c +0 -28
@@ -134,7 +134,7 @@ module Prism
134
134
  MINUS_GREATER: :tLAMBDA,
135
135
  NEWLINE: :tNL,
136
136
  NUMBERED_REFERENCE: :tNTH_REF,
137
- PARENTHESIS_LEFT: :tLPAREN,
137
+ PARENTHESIS_LEFT: :tLPAREN2,
138
138
  PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG,
139
139
  PARENTHESIS_RIGHT: :tRPAREN,
140
140
  PERCENT: :tPERCENT,
@@ -173,7 +173,7 @@ module Prism
173
173
  UMINUS_NUM: :tUNARY_NUM,
174
174
  UPLUS: :tUPLUS,
175
175
  USTAR: :tSTAR,
176
- USTAR_STAR: :tPOW,
176
+ USTAR_STAR: :tDSTAR,
177
177
  WORDS_SEP: :tSPACE
178
178
  }
179
179
 
@@ -187,7 +187,20 @@ module Prism
187
187
  EXPR_BEG = 0x1 # :nodoc:
188
188
  EXPR_LABEL = 0x400 # :nodoc:
189
189
 
190
- private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL
190
+ # It is used to determine whether `do` is of the token type `kDO` or `kDO_LAMBDA`.
191
+ #
192
+ # NOTE: In edge cases like `-> (foo = -> (bar) {}) do end`, please note that `kDO` is still returned
193
+ # instead of `kDO_LAMBDA`, which is expected: https://github.com/ruby/prism/pull/3046
194
+ LAMBDA_TOKEN_TYPES = [:kDO_LAMBDA, :tLAMBDA, :tLAMBEG]
195
+
196
+ # The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
197
+ # The following token types are listed as those classified as `tLPAREN`.
198
+ LPAREN_CONVERSION_TOKEN_TYPES = [
199
+ :kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
200
+ :tEQL, :tLPAREN, :tLPAREN2, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
201
+ ]
202
+
203
+ private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :LAMBDA_TOKEN_TYPES, :LPAREN_CONVERSION_TOKEN_TYPES
191
204
 
192
205
  # The Parser::Source::Buffer that the tokens were lexed from.
193
206
  attr_reader :source_buffer
@@ -229,6 +242,13 @@ module Prism
229
242
  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset])
230
243
 
231
244
  case type
245
+ when :kDO
246
+ types = tokens.map(&:first)
247
+ nearest_lambda_token_type = types.reverse.find { |type| LAMBDA_TOKEN_TYPES.include?(type) }
248
+
249
+ if nearest_lambda_token_type == :tLAMBDA
250
+ type = :kDO_LAMBDA
251
+ end
232
252
  when :tCHARACTER
233
253
  value.delete_prefix!("?")
234
254
  when :tCOMMENT
@@ -268,6 +288,8 @@ module Prism
268
288
  value.chomp!(":")
269
289
  when :tLCURLY
270
290
  type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
291
+ when :tLPAREN2
292
+ type = :tLPAREN if tokens.empty? || LPAREN_CONVERSION_TOKEN_TYPES.include?(tokens.dig(-1, 0))
271
293
  when :tNTH_REF
272
294
  value = parse_integer(value.delete_prefix("$"))
273
295
  when :tOP_ASGN
@@ -339,7 +361,7 @@ module Prism
339
361
  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
340
362
  end
341
363
  when :tSYMBEG
342
- if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
364
+ if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
343
365
  next_location = token.location.join(next_token.location)
344
366
  type = :tSYMBOL
345
367
  value = next_token.value
@@ -1,6 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "parser"
3
+ begin
4
+ require "parser"
5
+ rescue LoadError
6
+ warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.})
7
+ exit(1)
8
+ end
4
9
 
5
10
  module Prism
6
11
  module Translation
@@ -46,7 +51,7 @@ module Prism
46
51
  source = source_buffer.source
47
52
 
48
53
  offset_cache = build_offset_cache(source)
49
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
54
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
50
55
 
51
56
  build_ast(result.value, offset_cache)
52
57
  ensure
@@ -59,7 +64,7 @@ module Prism
59
64
  source = source_buffer.source
60
65
 
61
66
  offset_cache = build_offset_cache(source)
62
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
67
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
63
68
 
64
69
  [
65
70
  build_ast(result.value, offset_cache),
@@ -78,7 +83,7 @@ module Prism
78
83
  offset_cache = build_offset_cache(source)
79
84
  result =
80
85
  begin
81
- unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
86
+ unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
82
87
  rescue ::Parser::SyntaxError
83
88
  raise if !recover
84
89
  end
@@ -1273,8 +1273,8 @@ module Prism
1273
1273
  def visit_case_node(node)
1274
1274
  predicate = visit(node.predicate)
1275
1275
  clauses =
1276
- node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
1277
- on_when(*visit(condition), consequent)
1276
+ node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
1277
+ on_when(*visit(condition), current)
1278
1278
  end
1279
1279
 
1280
1280
  bounds(node.location)
@@ -1286,8 +1286,8 @@ module Prism
1286
1286
  def visit_case_match_node(node)
1287
1287
  predicate = visit(node.predicate)
1288
1288
  clauses =
1289
- node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
1290
- on_in(*visit(condition), consequent)
1289
+ node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
1290
+ on_in(*visit(condition), current)
1291
1291
  end
1292
1292
 
1293
1293
  bounds(node.location)
@@ -1908,7 +1908,7 @@ module Prism
1908
1908
  if node.then_keyword == "?"
1909
1909
  predicate = visit(node.predicate)
1910
1910
  truthy = visit(node.statements.body.first)
1911
- falsy = visit(node.consequent.statements.body.first)
1911
+ falsy = visit(node.subsequent.statements.body.first)
1912
1912
 
1913
1913
  bounds(node.location)
1914
1914
  on_ifop(predicate, truthy, falsy)
@@ -1921,13 +1921,13 @@ module Prism
1921
1921
  else
1922
1922
  visit(node.statements)
1923
1923
  end
1924
- consequent = visit(node.consequent)
1924
+ subsequent = visit(node.subsequent)
1925
1925
 
1926
1926
  bounds(node.location)
1927
1927
  if node.if_keyword == "if"
1928
- on_if(predicate, statements, consequent)
1928
+ on_if(predicate, statements, subsequent)
1929
1929
  else
1930
- on_elsif(predicate, statements, consequent)
1930
+ on_elsif(predicate, statements, subsequent)
1931
1931
  end
1932
1932
  else
1933
1933
  statements = visit(node.statements.body.first)
@@ -1960,7 +1960,7 @@ module Prism
1960
1960
  # ^^^^^^^^^^^^^^^^^^^^^
1961
1961
  def visit_in_node(node)
1962
1962
  # This is a special case where we're not going to call on_in directly
1963
- # because we don't have access to the consequent. Instead, we'll return
1963
+ # because we don't have access to the subsequent. Instead, we'll return
1964
1964
  # the component parts and let the parent node handle it.
1965
1965
  pattern = visit_pattern_node(node.pattern)
1966
1966
  statements =
@@ -2217,6 +2217,13 @@ module Prism
2217
2217
  end
2218
2218
  end
2219
2219
 
2220
+ # -> { it }
2221
+ # ^^
2222
+ def visit_it_local_variable_read_node(node)
2223
+ bounds(node.location)
2224
+ on_vcall(on_ident(node.slice))
2225
+ end
2226
+
2220
2227
  # -> { it }
2221
2228
  # ^^^^^^^^^
2222
2229
  def visit_it_parameters_node(node)
@@ -2312,12 +2319,7 @@ module Prism
2312
2319
  # ^^^
2313
2320
  def visit_local_variable_read_node(node)
2314
2321
  bounds(node.location)
2315
-
2316
- if node.name == :"0it"
2317
- on_vcall(on_ident(node.slice))
2318
- else
2319
- on_var_ref(on_ident(node.slice))
2320
- end
2322
+ on_var_ref(on_ident(node.slice))
2321
2323
  end
2322
2324
 
2323
2325
  # foo = 1
@@ -2806,10 +2808,10 @@ module Prism
2806
2808
  visit(node.statements)
2807
2809
  end
2808
2810
 
2809
- consequent = visit(node.consequent)
2811
+ subsequent = visit(node.subsequent)
2810
2812
 
2811
2813
  bounds(node.location)
2812
- on_rescue(exceptions, reference, statements, consequent)
2814
+ on_rescue(exceptions, reference, statements, subsequent)
2813
2815
  end
2814
2816
 
2815
2817
  # def foo(*bar); end
@@ -3130,10 +3132,10 @@ module Prism
3130
3132
  else
3131
3133
  visit(node.statements)
3132
3134
  end
3133
- consequent = visit(node.consequent)
3135
+ else_clause = visit(node.else_clause)
3134
3136
 
3135
3137
  bounds(node.location)
3136
- on_unless(predicate, statements, consequent)
3138
+ on_unless(predicate, statements, else_clause)
3137
3139
  else
3138
3140
  statements = visit(node.statements.body.first)
3139
3141
  predicate = visit(node.predicate)
@@ -3174,7 +3176,7 @@ module Prism
3174
3176
  # ^^^^^^^^^^^^^
3175
3177
  def visit_when_node(node)
3176
3178
  # This is a special case where we're not going to call on_when directly
3177
- # because we don't have access to the consequent. Instead, we'll return
3179
+ # because we don't have access to the subsequent. Instead, we'll return
3178
3180
  # the component parts and let the parent node handle it.
3179
3181
  conditions = visit_arguments(node.conditions)
3180
3182
  statements =
@@ -3267,11 +3269,7 @@ module Prism
3267
3269
 
3268
3270
  # Lazily initialize the parse result.
3269
3271
  def result
3270
- @result ||=
3271
- begin
3272
- scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]]
3273
- Prism.parse(source, scopes: scopes)
3274
- end
3272
+ @result ||= Prism.parse(source, partial_script: true)
3275
3273
  end
3276
3274
 
3277
3275
  ##########################################################################
@@ -1,6 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ruby_parser"
3
+ begin
4
+ require "ruby_parser"
5
+ rescue LoadError
6
+ warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.})
7
+ exit(1)
8
+ end
4
9
 
5
10
  module Prism
6
11
  module Translation
@@ -50,7 +55,19 @@ module Prism
50
55
  # a and b
51
56
  # ^^^^^^^
52
57
  def visit_and_node(node)
53
- s(node, :and, visit(node.left), visit(node.right))
58
+ left = visit(node.left)
59
+
60
+ if left[0] == :and
61
+ # ruby_parser has the and keyword as right-associative as opposed to
62
+ # prism which has it as left-associative. We reverse that
63
+ # associativity here.
64
+ nest = left
65
+ nest = nest[2] while nest[2][0] == :and
66
+ nest[2] = s(node, :and, nest[2], visit(node.right))
67
+ left
68
+ else
69
+ s(node, :and, left, visit(node.right))
70
+ end
54
71
  end
55
72
 
56
73
  # []
@@ -130,7 +147,7 @@ module Prism
130
147
  end
131
148
 
132
149
  current = node.rescue_clause
133
- until (current = current.consequent).nil?
150
+ until (current = current.subsequent).nil?
134
151
  result << visit(current)
135
152
  end
136
153
  end
@@ -246,6 +263,11 @@ module Prism
246
263
  when RegularExpressionNode, InterpolatedRegularExpressionNode
247
264
  return s(node, :match2, visit(node.receiver), visit(node.arguments.arguments.first))
248
265
  end
266
+
267
+ case node.arguments.arguments.first
268
+ when RegularExpressionNode, InterpolatedRegularExpressionNode
269
+ return s(node, :match3, visit(node.arguments.arguments.first), visit(node.receiver))
270
+ end
249
271
  end
250
272
  end
251
273
 
@@ -325,13 +347,13 @@ module Prism
325
347
  # case foo; when bar; end
326
348
  # ^^^^^^^^^^^^^^^^^^^^^^^
327
349
  def visit_case_node(node)
328
- s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
350
+ s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
329
351
  end
330
352
 
331
353
  # case foo; in bar; end
332
354
  # ^^^^^^^^^^^^^^^^^^^^^
333
355
  def visit_case_match_node(node)
334
- s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
356
+ s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
335
357
  end
336
358
 
337
359
  # class Foo; end
@@ -480,9 +502,9 @@ module Prism
480
502
  def visit_constant_path_target_node(node)
481
503
  inner =
482
504
  if node.parent.nil?
483
- s(node, :colon3, node.child.name)
505
+ s(node, :colon3, node.name)
484
506
  else
485
- s(node, :colon2, visit(node.parent), node.child.name)
507
+ s(node, :colon2, visit(node.parent), node.name)
486
508
  end
487
509
 
488
510
  s(node, :const, inner)
@@ -678,7 +700,7 @@ module Prism
678
700
  # foo ? bar : baz
679
701
  # ^^^^^^^^^^^^^^^
680
702
  def visit_if_node(node)
681
- s(node, :if, visit(node.predicate), visit(node.statements), visit(node.consequent))
703
+ s(node, :if, visit(node.predicate), visit(node.statements), visit(node.subsequent))
682
704
  end
683
705
 
684
706
  # 1i
@@ -859,6 +881,7 @@ module Prism
859
881
  # Visit the interpolated content of the string-like node.
860
882
  private def visit_interpolated_parts(parts)
861
883
  visited = []
884
+
862
885
  parts.each do |part|
863
886
  result = visit(part)
864
887
 
@@ -870,14 +893,25 @@ module Prism
870
893
  else
871
894
  visited << result
872
895
  end
896
+ visited << :space
897
+ elsif result[0] == :dstr
898
+ if !visited.empty? && part.parts[0].is_a?(StringNode)
899
+ # If we are in the middle of an implicitly concatenated string,
900
+ # we should not have a bare string as the first part. In this
901
+ # case we need to visit just that first part and then we can
902
+ # push the rest of the parts onto the visited array.
903
+ result[1] = visit(part.parts[0])
904
+ end
905
+ visited.concat(result[1..-1])
873
906
  else
874
907
  visited << result
875
908
  end
876
909
  end
877
910
 
878
911
  state = :beginning #: :beginning | :string_content | :interpolated_content
912
+ results = []
879
913
 
880
- visited.each_with_object([]) do |result, results|
914
+ visited.each_with_index do |result, index|
881
915
  case state
882
916
  when :beginning
883
917
  if result.is_a?(String)
@@ -892,7 +926,9 @@ module Prism
892
926
  state = :interpolated_content
893
927
  end
894
928
  when :string_content
895
- if result.is_a?(String)
929
+ if result == :space
930
+ # continue
931
+ elsif result.is_a?(String)
896
932
  results[0] << result
897
933
  elsif result.is_a?(Array) && result[0] == :str
898
934
  results[0] << result[1]
@@ -900,10 +936,25 @@ module Prism
900
936
  results << result
901
937
  state = :interpolated_content
902
938
  end
903
- else
904
- results << result
939
+ when :interpolated_content
940
+ if result == :space
941
+ # continue
942
+ elsif visited[index - 1] != :space && result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line)
943
+ results[-1][1] << result[1]
944
+ results[-1].line_max = result.line_max
945
+ else
946
+ results << result
947
+ end
905
948
  end
906
949
  end
950
+
951
+ results
952
+ end
953
+
954
+ # -> { it }
955
+ # ^^
956
+ def visit_it_local_variable_read_node(node)
957
+ s(node, :call, nil, :it)
907
958
  end
908
959
 
909
960
  # foo(bar: baz)
@@ -1118,7 +1169,19 @@ module Prism
1118
1169
  # a or b
1119
1170
  # ^^^^^^
1120
1171
  def visit_or_node(node)
1121
- s(node, :or, visit(node.left), visit(node.right))
1172
+ left = visit(node.left)
1173
+
1174
+ if left[0] == :or
1175
+ # ruby_parser has the or keyword as right-associative as opposed to
1176
+ # prism which has it as left-associative. We reverse that
1177
+ # associativity here.
1178
+ nest = left
1179
+ nest = nest[2] while nest[2][0] == :or
1180
+ nest[2] = s(node, :or, nest[2], visit(node.right))
1181
+ left
1182
+ else
1183
+ s(node, :or, left, visit(node.right))
1184
+ end
1122
1185
  end
1123
1186
 
1124
1187
  # def foo(bar, *baz); end
@@ -1374,7 +1437,13 @@ module Prism
1374
1437
  # "foo"
1375
1438
  # ^^^^^
1376
1439
  def visit_string_node(node)
1377
- s(node, :str, node.unescaped)
1440
+ unescaped = node.unescaped
1441
+
1442
+ if node.forced_binary_encoding?
1443
+ unescaped.force_encoding(Encoding::BINARY)
1444
+ end
1445
+
1446
+ s(node, :str, unescaped)
1378
1447
  end
1379
1448
 
1380
1449
  # super(foo)
@@ -1416,7 +1485,7 @@ module Prism
1416
1485
  # bar unless foo
1417
1486
  # ^^^^^^^^^^^^^^
1418
1487
  def visit_unless_node(node)
1419
- s(node, :if, visit(node.predicate), visit(node.consequent), visit(node.statements))
1488
+ s(node, :if, visit(node.predicate), visit(node.else_clause), visit(node.statements))
1420
1489
  end
1421
1490
 
1422
1491
  # until foo; bar end
@@ -1536,13 +1605,13 @@ module Prism
1536
1605
  # Parse the given source and translate it into the seattlerb/ruby_parser
1537
1606
  # gem's Sexp format.
1538
1607
  def parse(source, filepath = "(string)")
1539
- translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath)
1608
+ translate(Prism.parse(source, filepath: filepath, partial_script: true), filepath)
1540
1609
  end
1541
1610
 
1542
1611
  # Parse the given file and translate it into the seattlerb/ruby_parser
1543
1612
  # gem's Sexp format.
1544
1613
  def parse_file(filepath)
1545
- translate(Prism.parse_file(filepath, scopes: [[]]), filepath)
1614
+ translate(Prism.parse_file(filepath, partial_script: true), filepath)
1546
1615
  end
1547
1616
 
1548
1617
  class << self
data/lib/prism/visitor.rb CHANGED
@@ -313,6 +313,9 @@ module Prism
313
313
  # Visit a InterpolatedXStringNode node
314
314
  alias visit_interpolated_x_string_node visit_child_nodes
315
315
 
316
+ # Visit a ItLocalVariableReadNode node
317
+ alias visit_it_local_variable_read_node visit_child_nodes
318
+
316
319
  # Visit a ItParametersNode node
317
320
  alias visit_it_parameters_node visit_child_nodes
318
321
 
data/lib/prism.rb CHANGED
@@ -13,7 +13,6 @@ module Prism
13
13
 
14
14
  autoload :BasicVisitor, "prism/visitor"
15
15
  autoload :Compiler, "prism/compiler"
16
- autoload :Debug, "prism/debug"
17
16
  autoload :DesugarCompiler, "prism/desugar_compiler"
18
17
  autoload :Dispatcher, "prism/dispatcher"
19
18
  autoload :DotVisitor, "prism/dot_visitor"
@@ -25,14 +24,15 @@ module Prism
25
24
  autoload :Pack, "prism/pack"
26
25
  autoload :Pattern, "prism/pattern"
27
26
  autoload :Reflection, "prism/reflection"
27
+ autoload :Relocation, "prism/relocation"
28
28
  autoload :Serialize, "prism/serialize"
29
+ autoload :StringQuery, "prism/string_query"
29
30
  autoload :Translation, "prism/translation"
30
31
  autoload :Visitor, "prism/visitor"
31
32
 
32
33
  # Some of these constants are not meant to be exposed, so marking them as
33
34
  # private here.
34
35
 
35
- private_constant :Debug
36
36
  private_constant :LexCompat
37
37
  private_constant :LexRipper
38
38
 
@@ -71,21 +71,19 @@ require_relative "prism/polyfill/byteindex"
71
71
  require_relative "prism/node"
72
72
  require_relative "prism/node_ext"
73
73
  require_relative "prism/parse_result"
74
- require_relative "prism/parse_result/comments"
75
- require_relative "prism/parse_result/newlines"
76
74
 
77
75
  # This is a Ruby implementation of the prism parser. If we're running on CRuby
78
76
  # and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then
79
77
  # it's going to require the built library. Otherwise, it's going to require a
80
78
  # module that uses FFI to call into the library.
81
79
  if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
82
- require "prism/prism"
83
-
84
80
  # The C extension is the default backend on CRuby.
85
81
  Prism::BACKEND = :CEXT
86
- else
87
- require_relative "prism/ffi"
88
82
 
83
+ require "prism/prism"
84
+ else
89
85
  # The FFI backend is used on other Ruby implementations.
90
86
  Prism::BACKEND = :FFI
87
+
88
+ require_relative "prism/ffi"
91
89
  end
data/prism.gemspec CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |spec|
4
4
  spec.name = "prism"
5
- spec.version = "0.29.0"
5
+ spec.version = "1.3.0"
6
6
  spec.authors = ["Shopify"]
7
7
  spec.email = ["ruby@shopify.com"]
8
8
 
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
35
35
  "docs/parser_translation.md",
36
36
  "docs/parsing_rules.md",
37
37
  "docs/releasing.md",
38
+ "docs/relocation.md",
38
39
  "docs/ripper_translation.md",
39
40
  "docs/ruby_api.md",
40
41
  "docs/ruby_parser_translation.md",
@@ -65,12 +66,10 @@ Gem::Specification.new do |spec|
65
66
  "include/prism/util/pm_newline_list.h",
66
67
  "include/prism/util/pm_strncasecmp.h",
67
68
  "include/prism/util/pm_string.h",
68
- "include/prism/util/pm_string_list.h",
69
69
  "include/prism/util/pm_strpbrk.h",
70
70
  "include/prism/version.h",
71
71
  "lib/prism.rb",
72
72
  "lib/prism/compiler.rb",
73
- "lib/prism/debug.rb",
74
73
  "lib/prism/desugar_compiler.rb",
75
74
  "lib/prism/dispatcher.rb",
76
75
  "lib/prism/dot_visitor.rb",
@@ -84,19 +83,21 @@ Gem::Specification.new do |spec|
84
83
  "lib/prism/pack.rb",
85
84
  "lib/prism/parse_result.rb",
86
85
  "lib/prism/parse_result/comments.rb",
86
+ "lib/prism/parse_result/errors.rb",
87
87
  "lib/prism/parse_result/newlines.rb",
88
88
  "lib/prism/pattern.rb",
89
89
  "lib/prism/polyfill/byteindex.rb",
90
90
  "lib/prism/polyfill/unpack1.rb",
91
91
  "lib/prism/reflection.rb",
92
+ "lib/prism/relocation.rb",
92
93
  "lib/prism/serialize.rb",
94
+ "lib/prism/string_query.rb",
93
95
  "lib/prism/translation.rb",
94
96
  "lib/prism/translation/parser.rb",
95
97
  "lib/prism/translation/parser33.rb",
96
98
  "lib/prism/translation/parser34.rb",
97
99
  "lib/prism/translation/parser/compiler.rb",
98
100
  "lib/prism/translation/parser/lexer.rb",
99
- "lib/prism/translation/parser/rubocop.rb",
100
101
  "lib/prism/translation/ripper.rb",
101
102
  "lib/prism/translation/ripper/sexp.rb",
102
103
  "lib/prism/translation/ripper/shim.rb",
@@ -105,11 +106,13 @@ Gem::Specification.new do |spec|
105
106
  "prism.gemspec",
106
107
  "rbi/prism.rbi",
107
108
  "rbi/prism/compiler.rbi",
109
+ "rbi/prism/dsl.rbi",
108
110
  "rbi/prism/inspect_visitor.rbi",
109
111
  "rbi/prism/node_ext.rbi",
110
112
  "rbi/prism/node.rbi",
111
113
  "rbi/prism/parse_result.rbi",
112
114
  "rbi/prism/reflection.rbi",
115
+ "rbi/prism/string_query.rbi",
113
116
  "rbi/prism/translation/parser.rbi",
114
117
  "rbi/prism/translation/parser33.rbi",
115
118
  "rbi/prism/translation/parser34.rbi",
@@ -129,7 +132,9 @@ Gem::Specification.new do |spec|
129
132
  "sig/prism/parse_result.rbs",
130
133
  "sig/prism/pattern.rbs",
131
134
  "sig/prism/reflection.rbs",
135
+ "sig/prism/relocation.rbs",
132
136
  "sig/prism/serialize.rbs",
137
+ "sig/prism/string_query.rbs",
133
138
  "sig/prism/visitor.rbs",
134
139
  "src/diagnostic.c",
135
140
  "src/encoding.c",
@@ -149,7 +154,6 @@ Gem::Specification.new do |spec|
149
154
  "src/util/pm_list.c",
150
155
  "src/util/pm_memchr.c",
151
156
  "src/util/pm_newline_list.c",
152
- "src/util/pm_string_list.c",
153
157
  "src/util/pm_string.c",
154
158
  "src/util/pm_strncasecmp.c",
155
159
  "src/util/pm_strpbrk.c"