prism 0.24.0 → 0.29.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/BSDmakefile +58 -0
  3. data/CHANGELOG.md +132 -1
  4. data/Makefile +25 -18
  5. data/README.md +45 -6
  6. data/config.yml +828 -25
  7. data/docs/build_system.md +31 -0
  8. data/docs/configuration.md +4 -0
  9. data/docs/cruby_compilation.md +1 -1
  10. data/docs/parser_translation.md +14 -9
  11. data/docs/releasing.md +7 -9
  12. data/docs/ripper_translation.md +50 -0
  13. data/docs/ruby_api.md +1 -0
  14. data/docs/serialization.md +26 -5
  15. data/ext/prism/api_node.c +1037 -936
  16. data/ext/prism/api_pack.c +9 -0
  17. data/ext/prism/extconf.rb +62 -18
  18. data/ext/prism/extension.c +351 -71
  19. data/ext/prism/extension.h +5 -4
  20. data/include/prism/ast.h +539 -101
  21. data/include/prism/defines.h +106 -2
  22. data/include/prism/diagnostic.h +168 -74
  23. data/include/prism/encoding.h +22 -4
  24. data/include/prism/node.h +93 -0
  25. data/include/prism/options.h +84 -9
  26. data/include/prism/pack.h +11 -0
  27. data/include/prism/parser.h +213 -54
  28. data/include/prism/prettyprint.h +8 -0
  29. data/include/prism/static_literals.h +120 -0
  30. data/include/prism/util/pm_buffer.h +65 -2
  31. data/include/prism/util/pm_constant_pool.h +18 -1
  32. data/include/prism/util/pm_integer.h +119 -0
  33. data/include/prism/util/pm_list.h +1 -1
  34. data/include/prism/util/pm_newline_list.h +8 -0
  35. data/include/prism/util/pm_string.h +26 -2
  36. data/include/prism/version.h +2 -2
  37. data/include/prism.h +59 -1
  38. data/lib/prism/compiler.rb +8 -1
  39. data/lib/prism/debug.rb +46 -3
  40. data/lib/prism/desugar_compiler.rb +5 -3
  41. data/lib/prism/dispatcher.rb +29 -0
  42. data/lib/prism/dot_visitor.rb +141 -54
  43. data/lib/prism/dsl.rb +48 -36
  44. data/lib/prism/ffi.rb +82 -17
  45. data/lib/prism/inspect_visitor.rb +2156 -0
  46. data/lib/prism/lex_compat.rb +34 -15
  47. data/lib/prism/mutation_compiler.rb +13 -2
  48. data/lib/prism/node.rb +4453 -4459
  49. data/lib/prism/node_ext.rb +249 -30
  50. data/lib/prism/pack.rb +4 -0
  51. data/lib/prism/parse_result/comments.rb +35 -18
  52. data/lib/prism/parse_result/newlines.rb +2 -2
  53. data/lib/prism/parse_result.rb +218 -43
  54. data/lib/prism/pattern.rb +28 -10
  55. data/lib/prism/polyfill/byteindex.rb +13 -0
  56. data/lib/prism/polyfill/unpack1.rb +14 -0
  57. data/lib/prism/reflection.rb +411 -0
  58. data/lib/prism/serialize.rb +480 -112
  59. data/lib/prism/translation/parser/compiler.rb +376 -88
  60. data/lib/prism/translation/parser/lexer.rb +103 -22
  61. data/lib/prism/translation/parser/rubocop.rb +41 -13
  62. data/lib/prism/translation/parser.rb +123 -11
  63. data/lib/prism/translation/parser33.rb +1 -1
  64. data/lib/prism/translation/parser34.rb +1 -1
  65. data/lib/prism/translation/ripper/sexp.rb +125 -0
  66. data/lib/prism/translation/ripper/shim.rb +5 -0
  67. data/lib/prism/translation/ripper.rb +3216 -462
  68. data/lib/prism/translation/ruby_parser.rb +111 -56
  69. data/lib/prism/translation.rb +3 -1
  70. data/lib/prism/visitor.rb +10 -0
  71. data/lib/prism.rb +12 -20
  72. data/prism.gemspec +46 -14
  73. data/rbi/prism/compiler.rbi +12 -0
  74. data/rbi/prism/inspect_visitor.rbi +12 -0
  75. data/rbi/prism/node.rbi +8712 -0
  76. data/rbi/prism/node_ext.rbi +107 -0
  77. data/rbi/prism/parse_result.rbi +358 -0
  78. data/rbi/prism/reflection.rbi +58 -0
  79. data/rbi/prism/translation/parser.rbi +11 -0
  80. data/rbi/prism/translation/parser33.rbi +6 -0
  81. data/rbi/prism/translation/parser34.rbi +6 -0
  82. data/rbi/prism/translation/ripper.rbi +15 -0
  83. data/rbi/prism/visitor.rbi +470 -0
  84. data/rbi/prism.rbi +38 -7748
  85. data/sig/prism/compiler.rbs +9 -0
  86. data/sig/prism/dispatcher.rbs +16 -0
  87. data/sig/prism/dot_visitor.rbs +6 -0
  88. data/sig/prism/dsl.rbs +462 -0
  89. data/sig/prism/inspect_visitor.rbs +22 -0
  90. data/sig/prism/lex_compat.rbs +10 -0
  91. data/sig/prism/mutation_compiler.rbs +158 -0
  92. data/sig/prism/node.rbs +3558 -0
  93. data/sig/prism/node_ext.rbs +82 -0
  94. data/sig/prism/pack.rbs +43 -0
  95. data/sig/prism/parse_result.rbs +160 -0
  96. data/sig/prism/pattern.rbs +13 -0
  97. data/sig/prism/reflection.rbs +50 -0
  98. data/sig/prism/serialize.rbs +6 -0
  99. data/sig/prism/visitor.rbs +168 -0
  100. data/sig/prism.rbs +188 -4767
  101. data/src/diagnostic.c +636 -230
  102. data/src/encoding.c +211 -108
  103. data/src/node.c +7555 -451
  104. data/src/options.c +66 -31
  105. data/src/pack.c +33 -17
  106. data/src/prettyprint.c +1383 -1431
  107. data/src/prism.c +4734 -1310
  108. data/src/regexp.c +17 -2
  109. data/src/serialize.c +68 -46
  110. data/src/static_literals.c +638 -0
  111. data/src/token_type.c +10 -9
  112. data/src/util/pm_buffer.c +147 -20
  113. data/src/util/pm_char.c +4 -4
  114. data/src/util/pm_constant_pool.c +35 -11
  115. data/src/util/pm_integer.c +642 -0
  116. data/src/util/pm_list.c +1 -1
  117. data/src/util/pm_newline_list.c +14 -5
  118. data/src/util/pm_string.c +134 -5
  119. data/src/util/pm_string_list.c +2 -2
  120. metadata +41 -9
  121. data/docs/ripper.md +0 -36
  122. data/include/prism/util/pm_state_stack.h +0 -42
  123. data/lib/prism/node_inspector.rb +0 -68
  124. data/rbi/prism_static.rbi +0 -207
  125. data/sig/prism_static.rbs +0 -201
  126. data/src/util/pm_state_stack.c +0 -25
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  =begin
3
4
  This file is generated by the templates/template.rb script and should not be
4
5
  modified manually. See templates/lib/prism/serialize.rb.erb
@@ -6,17 +7,7 @@ if you are looking to modify the template
6
7
  =end
7
8
 
8
9
  require "stringio"
9
-
10
- # Polyfill for String#unpack1 with the offset parameter.
11
- if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
12
- String.prepend(
13
- Module.new {
14
- def unpack1(format, offset: 0) # :nodoc:
15
- offset == 0 ? super(format) : self[offset..].unpack1(format)
16
- end
17
- }
18
- )
19
- end
10
+ require_relative "polyfill/unpack1"
20
11
 
21
12
  module Prism
22
13
  # A module responsible for deserializing parse results.
@@ -27,7 +18,7 @@ module Prism
27
18
 
28
19
  # The minor version of prism that we are expecting to find in the serialized
29
20
  # strings.
30
- MINOR_VERSION = 24
21
+ MINOR_VERSION = 29
31
22
 
32
23
  # The patch version of prism that we are expecting to find in the serialized
33
24
  # strings.
@@ -36,7 +27,7 @@ module Prism
36
27
  # Deserialize the AST represented by the given string into a parse result.
37
28
  def self.load(input, serialized)
38
29
  input = input.dup
39
- source = Source.new(input)
30
+ source = Source.for(input)
40
31
  loader = Loader.new(source, serialized)
41
32
  result = loader.load_result
42
33
 
@@ -51,6 +42,37 @@ module Prism
51
42
  end
52
43
 
53
44
  class Loader # :nodoc:
45
+ if RUBY_ENGINE == "truffleruby"
46
+ # StringIO is synchronized and that adds a high overhead on TruffleRuby.
47
+ class FastStringIO # :nodoc:
48
+ attr_accessor :pos
49
+
50
+ def initialize(string)
51
+ @string = string
52
+ @pos = 0
53
+ end
54
+
55
+ def getbyte
56
+ byte = @string.getbyte(@pos)
57
+ @pos += 1
58
+ byte
59
+ end
60
+
61
+ def read(n)
62
+ slice = @string.byteslice(@pos, n)
63
+ @pos += n
64
+ slice
65
+ end
66
+
67
+ def eof?
68
+ @pos >= @string.bytesize
69
+ end
70
+ end
71
+ else
72
+ FastStringIO = ::StringIO
73
+ end
74
+ private_constant :FastStringIO
75
+
54
76
  attr_reader :encoding, :input, :serialized, :io
55
77
  attr_reader :constant_pool_offset, :constant_pool, :source
56
78
  attr_reader :start_line
@@ -59,9 +81,9 @@ module Prism
59
81
  @encoding = Encoding::UTF_8
60
82
 
61
83
  @input = source.source.dup
84
+ raise unless serialized.encoding == Encoding::BINARY
62
85
  @serialized = serialized
63
- @io = StringIO.new(serialized)
64
- @io.set_encoding(Encoding::BINARY)
86
+ @io = FastStringIO.new(serialized)
65
87
 
66
88
  @constant_pool_offset = nil
67
89
  @constant_pool = nil
@@ -73,7 +95,7 @@ module Prism
73
95
  def load_header
74
96
  raise "Invalid serialization" if io.read(5) != "PRISM"
75
97
  raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
76
- only_semantic_fields = io.read(1).unpack1("C")
98
+ only_semantic_fields = io.getbyte
77
99
  unless only_semantic_fields == 0
78
100
  raise "Invalid serialization (location fields must be included but are not)"
79
101
  end
@@ -96,19 +118,324 @@ module Prism
96
118
  def load_comments
97
119
  Array.new(load_varuint) do
98
120
  case load_varuint
99
- when 0 then InlineComment.new(load_location)
100
- when 1 then EmbDocComment.new(load_location)
101
- when 2 then DATAComment.new(load_location)
121
+ when 0 then InlineComment.new(load_location_object)
122
+ when 1 then EmbDocComment.new(load_location_object)
102
123
  end
103
124
  end
104
125
  end
105
126
 
127
+ DIAGNOSTIC_TYPES = [
128
+ :alias_argument,
129
+ :alias_argument_numbered_reference,
130
+ :ampampeq_multi_assign,
131
+ :argument_after_block,
132
+ :argument_after_forwarding_ellipses,
133
+ :argument_bare_hash,
134
+ :argument_block_forwarding,
135
+ :argument_block_multi,
136
+ :argument_conflict_ampersand,
137
+ :argument_conflict_star,
138
+ :argument_conflict_star_star,
139
+ :argument_formal_class,
140
+ :argument_formal_constant,
141
+ :argument_formal_global,
142
+ :argument_formal_ivar,
143
+ :argument_forwarding_unbound,
144
+ :argument_in,
145
+ :argument_no_forwarding_ampersand,
146
+ :argument_no_forwarding_ellipses,
147
+ :argument_no_forwarding_star,
148
+ :argument_no_forwarding_star_star,
149
+ :argument_splat_after_assoc_splat,
150
+ :argument_splat_after_splat,
151
+ :argument_term_paren,
152
+ :argument_unexpected_block,
153
+ :array_element,
154
+ :array_expression,
155
+ :array_expression_after_star,
156
+ :array_separator,
157
+ :array_term,
158
+ :begin_lonely_else,
159
+ :begin_term,
160
+ :begin_upcase_brace,
161
+ :begin_upcase_term,
162
+ :begin_upcase_toplevel,
163
+ :block_param_local_variable,
164
+ :block_param_pipe_term,
165
+ :block_term_brace,
166
+ :block_term_end,
167
+ :cannot_parse_expression,
168
+ :cannot_parse_string_part,
169
+ :case_expression_after_case,
170
+ :case_expression_after_when,
171
+ :case_match_missing_predicate,
172
+ :case_missing_conditions,
173
+ :case_term,
174
+ :class_in_method,
175
+ :class_name,
176
+ :class_superclass,
177
+ :class_term,
178
+ :class_unexpected_end,
179
+ :class_variable_bare,
180
+ :conditional_elsif_predicate,
181
+ :conditional_if_predicate,
182
+ :conditional_predicate_term,
183
+ :conditional_term,
184
+ :conditional_term_else,
185
+ :conditional_unless_predicate,
186
+ :conditional_until_predicate,
187
+ :conditional_while_predicate,
188
+ :constant_path_colon_colon_constant,
189
+ :def_endless,
190
+ :def_endless_setter,
191
+ :def_name,
192
+ :def_params_term,
193
+ :def_params_term_paren,
194
+ :def_receiver,
195
+ :def_receiver_term,
196
+ :def_term,
197
+ :defined_expression,
198
+ :embdoc_term,
199
+ :embexpr_end,
200
+ :embvar_invalid,
201
+ :end_upcase_brace,
202
+ :end_upcase_term,
203
+ :escape_invalid_control,
204
+ :escape_invalid_control_repeat,
205
+ :escape_invalid_hexadecimal,
206
+ :escape_invalid_meta,
207
+ :escape_invalid_meta_repeat,
208
+ :escape_invalid_unicode,
209
+ :escape_invalid_unicode_cm_flags,
210
+ :escape_invalid_unicode_literal,
211
+ :escape_invalid_unicode_long,
212
+ :escape_invalid_unicode_term,
213
+ :expect_argument,
214
+ :expect_eol_after_statement,
215
+ :expect_expression_after_ampampeq,
216
+ :expect_expression_after_comma,
217
+ :expect_expression_after_equal,
218
+ :expect_expression_after_less_less,
219
+ :expect_expression_after_lparen,
220
+ :expect_expression_after_operator,
221
+ :expect_expression_after_pipepipeeq,
222
+ :expect_expression_after_question,
223
+ :expect_expression_after_splat,
224
+ :expect_expression_after_splat_hash,
225
+ :expect_expression_after_star,
226
+ :expect_ident_req_parameter,
227
+ :expect_in_delimiter,
228
+ :expect_lparen_req_parameter,
229
+ :expect_message,
230
+ :expect_rbracket,
231
+ :expect_rparen,
232
+ :expect_rparen_after_multi,
233
+ :expect_rparen_req_parameter,
234
+ :expect_string_content,
235
+ :expect_when_delimiter,
236
+ :expression_bare_hash,
237
+ :expression_not_writable,
238
+ :expression_not_writable_encoding,
239
+ :expression_not_writable_false,
240
+ :expression_not_writable_file,
241
+ :expression_not_writable_line,
242
+ :expression_not_writable_nil,
243
+ :expression_not_writable_self,
244
+ :expression_not_writable_true,
245
+ :float_parse,
246
+ :for_collection,
247
+ :for_in,
248
+ :for_index,
249
+ :for_term,
250
+ :global_variable_bare,
251
+ :hash_expression_after_label,
252
+ :hash_key,
253
+ :hash_rocket,
254
+ :hash_term,
255
+ :hash_value,
256
+ :heredoc_identifier,
257
+ :heredoc_term,
258
+ :incomplete_question_mark,
259
+ :incomplete_variable_class,
260
+ :incomplete_variable_class_3_3,
261
+ :incomplete_variable_instance,
262
+ :incomplete_variable_instance_3_3,
263
+ :instance_variable_bare,
264
+ :invalid_block_exit,
265
+ :invalid_character,
266
+ :invalid_encoding_magic_comment,
267
+ :invalid_escape_character,
268
+ :invalid_float_exponent,
269
+ :invalid_local_variable_read,
270
+ :invalid_local_variable_write,
271
+ :invalid_multibyte_char,
272
+ :invalid_multibyte_character,
273
+ :invalid_multibyte_escape,
274
+ :invalid_number_binary,
275
+ :invalid_number_decimal,
276
+ :invalid_number_fraction,
277
+ :invalid_number_hexadecimal,
278
+ :invalid_number_octal,
279
+ :invalid_number_underscore_inner,
280
+ :invalid_number_underscore_trailing,
281
+ :invalid_percent,
282
+ :invalid_printable_character,
283
+ :invalid_retry_after_else,
284
+ :invalid_retry_after_ensure,
285
+ :invalid_retry_without_rescue,
286
+ :invalid_symbol,
287
+ :invalid_variable_global,
288
+ :invalid_variable_global_3_3,
289
+ :invalid_yield,
290
+ :it_not_allowed_numbered,
291
+ :it_not_allowed_ordinary,
292
+ :lambda_open,
293
+ :lambda_term_brace,
294
+ :lambda_term_end,
295
+ :list_i_lower_element,
296
+ :list_i_lower_term,
297
+ :list_i_upper_element,
298
+ :list_i_upper_term,
299
+ :list_w_lower_element,
300
+ :list_w_lower_term,
301
+ :list_w_upper_element,
302
+ :list_w_upper_term,
303
+ :malloc_failed,
304
+ :mixed_encoding,
305
+ :module_in_method,
306
+ :module_name,
307
+ :module_term,
308
+ :multi_assign_multi_splats,
309
+ :multi_assign_unexpected_rest,
310
+ :no_local_variable,
311
+ :not_expression,
312
+ :number_literal_underscore,
313
+ :numbered_parameter_it,
314
+ :numbered_parameter_ordinary,
315
+ :numbered_parameter_outer_scope,
316
+ :operator_multi_assign,
317
+ :operator_write_arguments,
318
+ :operator_write_block,
319
+ :parameter_assoc_splat_multi,
320
+ :parameter_block_multi,
321
+ :parameter_circular,
322
+ :parameter_forwarding_after_rest,
323
+ :parameter_method_name,
324
+ :parameter_name_duplicated,
325
+ :parameter_no_default,
326
+ :parameter_no_default_kw,
327
+ :parameter_numbered_reserved,
328
+ :parameter_order,
329
+ :parameter_splat_multi,
330
+ :parameter_star,
331
+ :parameter_unexpected_fwd,
332
+ :parameter_wild_loose_comma,
333
+ :parameter_unexpected_no_kw,
334
+ :pattern_capture_duplicate,
335
+ :pattern_expression_after_bracket,
336
+ :pattern_expression_after_comma,
337
+ :pattern_expression_after_hrocket,
338
+ :pattern_expression_after_in,
339
+ :pattern_expression_after_key,
340
+ :pattern_expression_after_paren,
341
+ :pattern_expression_after_pin,
342
+ :pattern_expression_after_pipe,
343
+ :pattern_expression_after_range,
344
+ :pattern_expression_after_rest,
345
+ :pattern_hash_implicit,
346
+ :pattern_hash_key,
347
+ :pattern_hash_key_duplicate,
348
+ :pattern_hash_key_interpolated,
349
+ :pattern_hash_key_label,
350
+ :pattern_hash_key_locals,
351
+ :pattern_ident_after_hrocket,
352
+ :pattern_label_after_comma,
353
+ :pattern_rest,
354
+ :pattern_term_brace,
355
+ :pattern_term_bracket,
356
+ :pattern_term_paren,
357
+ :pipepipeeq_multi_assign,
358
+ :regexp_encoding_option_mismatch,
359
+ :regexp_incompat_char_encoding,
360
+ :regexp_invalid_unicode_range,
361
+ :regexp_non_escaped_mbc,
362
+ :regexp_term,
363
+ :regexp_unknown_options,
364
+ :regexp_utf8_char_non_utf8_regexp,
365
+ :rescue_expression,
366
+ :rescue_modifier_value,
367
+ :rescue_term,
368
+ :rescue_variable,
369
+ :return_invalid,
370
+ :script_not_found,
371
+ :singleton_for_literals,
372
+ :statement_alias,
373
+ :statement_postexe_end,
374
+ :statement_preexe_begin,
375
+ :statement_undef,
376
+ :string_concatenation,
377
+ :string_interpolated_term,
378
+ :string_literal_eof,
379
+ :string_literal_term,
380
+ :symbol_invalid,
381
+ :symbol_term_dynamic,
382
+ :symbol_term_interpolated,
383
+ :ternary_colon,
384
+ :ternary_expression_false,
385
+ :ternary_expression_true,
386
+ :unary_receiver,
387
+ :undef_argument,
388
+ :unexpected_block_argument,
389
+ :unexpected_index_block,
390
+ :unexpected_index_keywords,
391
+ :unexpected_safe_navigation,
392
+ :unexpected_token_close_context,
393
+ :unexpected_token_ignore,
394
+ :until_term,
395
+ :void_expression,
396
+ :while_term,
397
+ :write_target_in_method,
398
+ :write_target_readonly,
399
+ :write_target_unexpected,
400
+ :xstring_term,
401
+ :ambiguous_first_argument_minus,
402
+ :ambiguous_first_argument_plus,
403
+ :ambiguous_prefix_ampersand,
404
+ :ambiguous_prefix_star,
405
+ :ambiguous_prefix_star_star,
406
+ :ambiguous_slash,
407
+ :comparison_after_comparison,
408
+ :dot_dot_dot_eol,
409
+ :equal_in_conditional,
410
+ :equal_in_conditional_3_3,
411
+ :end_in_method,
412
+ :duplicated_hash_key,
413
+ :duplicated_when_clause,
414
+ :float_out_of_range,
415
+ :ignored_frozen_string_literal,
416
+ :integer_in_flip_flop,
417
+ :invalid_character,
418
+ :invalid_numbered_reference,
419
+ :invalid_shareable_constant_value,
420
+ :keyword_eol,
421
+ :literal_in_condition_default,
422
+ :literal_in_condition_verbose,
423
+ :shareable_constant_value_line,
424
+ :shebang_carriage_return,
425
+ :unexpected_carriage_return,
426
+ :unreachable_statement,
427
+ :unused_local_variable,
428
+ :void_statement,
429
+ ].freeze
430
+
431
+ private_constant :DIAGNOSTIC_TYPES
432
+
106
433
  def load_metadata
107
434
  comments = load_comments
108
- magic_comments = Array.new(load_varuint) { MagicComment.new(load_location, load_location) }
109
- data_loc = load_optional_location
110
- errors = Array.new(load_varuint) { ParseError.new(load_embedded_string, load_location, load_error_level) }
111
- warnings = Array.new(load_varuint) { ParseWarning.new(load_embedded_string, load_location, load_warning_level) }
435
+ magic_comments = Array.new(load_varuint) { MagicComment.new(load_location_object, load_location_object) }
436
+ data_loc = load_optional_location_object
437
+ errors = Array.new(load_varuint) { ParseError.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_error_level) }
438
+ warnings = Array.new(load_varuint) { ParseWarning.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_warning_level) }
112
439
  [comments, magic_comments, data_loc, errors, warnings]
113
440
  end
114
441
 
@@ -119,7 +446,7 @@ module Prism
119
446
  length = load_varuint
120
447
  lex_state = load_varuint
121
448
  location = Location.new(@source, start, length)
122
- tokens << [Prism::Token.new(source, type, location.slice, location), lex_state]
449
+ tokens << [Token.new(source, type, location.slice, location), lex_state]
123
450
  end
124
451
 
125
452
  tokens
@@ -134,7 +461,7 @@ module Prism
134
461
  tokens.each { |token,| token.value.force_encoding(encoding) }
135
462
 
136
463
  raise "Expected to consume all bytes while deserializing" unless @io.eof?
137
- Prism::ParseResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, @source)
464
+ LexResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, @source)
138
465
  end
139
466
 
140
467
  def load_nodes
@@ -145,7 +472,7 @@ module Prism
145
472
 
146
473
  comments, magic_comments, data_loc, errors, warnings = load_metadata
147
474
 
148
- @constant_pool_offset = io.read(4).unpack1("L")
475
+ @constant_pool_offset = load_uint32
149
476
  @constant_pool = Array.new(load_varuint, nil)
150
477
 
151
478
  [load_node, comments, magic_comments, data_loc, errors, warnings]
@@ -153,7 +480,7 @@ module Prism
153
480
 
154
481
  def load_result
155
482
  node, comments, magic_comments, data_loc, errors, warnings = load_nodes
156
- Prism::ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, @source)
483
+ ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, @source)
157
484
  end
158
485
 
159
486
  private
@@ -179,7 +506,22 @@ module Prism
179
506
  (n >> 1) ^ (-(n & 1))
180
507
  end
181
508
 
182
- def load_serialized_length
509
+ def load_integer
510
+ negative = io.getbyte != 0
511
+ length = load_varuint
512
+
513
+ value = 0
514
+ length.times { |index| value |= (load_varuint << (index * 32)) }
515
+
516
+ value = -value if negative
517
+ value
518
+ end
519
+
520
+ def load_double
521
+ io.read(8).unpack1("D")
522
+ end
523
+
524
+ def load_uint32
183
525
  io.read(4).unpack1("L")
184
526
  end
185
527
 
@@ -207,6 +549,10 @@ module Prism
207
549
  end
208
550
 
209
551
  def load_location
552
+ (load_varuint << 32) | load_varuint
553
+ end
554
+
555
+ def load_location_object
210
556
  Location.new(source, load_varuint, load_varuint)
211
557
  end
212
558
 
@@ -214,19 +560,23 @@ module Prism
214
560
  load_location if io.getbyte != 0
215
561
  end
216
562
 
563
+ def load_optional_location_object
564
+ load_location_object if io.getbyte != 0
565
+ end
566
+
217
567
  def load_constant(index)
218
568
  constant = constant_pool[index]
219
569
 
220
570
  unless constant
221
571
  offset = constant_pool_offset + index * 8
222
- start = serialized.unpack1("L", offset: offset)
223
- length = serialized.unpack1("L", offset: offset + 4)
572
+ start = @serialized.unpack1("L", offset: offset)
573
+ length = @serialized.unpack1("L", offset: offset + 4)
224
574
 
225
575
  constant =
226
576
  if start.nobits?(1 << 31)
227
577
  input.byteslice(start, length).force_encoding(@encoding).to_sym
228
578
  else
229
- serialized.byteslice(start & ((1 << 31) - 1), length).force_encoding(@encoding).to_sym
579
+ @serialized.byteslice(start & ((1 << 31) - 1), length).force_encoding(@encoding).to_sym
230
580
  end
231
581
 
232
582
  constant_pool[index] = constant
@@ -249,9 +599,11 @@ module Prism
249
599
 
250
600
  case level
251
601
  when 0
252
- :fatal
602
+ :syntax
253
603
  when 1
254
604
  :argument
605
+ when 2
606
+ :load
255
607
  else
256
608
  raise "Unknown level: #{level}"
257
609
  end
@@ -371,7 +723,7 @@ module Prism
371
723
  source, load_required_constant, location)
372
724
  when 32 then
373
725
  ClassVariableWriteNode.new(
374
- source, load_required_constant, load_location, load_node, load_optional_location, location)
726
+ source, load_required_constant, load_location, load_node, load_location, location)
375
727
  when 33 then
376
728
  ConstantAndWriteNode.new(
377
729
  source, load_required_constant, load_location, load_location, load_node, location)
@@ -386,7 +738,7 @@ module Prism
386
738
  source, load_node, load_location, load_node, location)
387
739
  when 37 then
388
740
  ConstantPathNode.new(
389
- source, load_optional_node, load_node, load_location, location)
741
+ source, load_optional_node, load_optional_constant, load_location, load_location, location)
390
742
  when 38 then
391
743
  ConstantPathOperatorWriteNode.new(
392
744
  source, load_node, load_location, load_node, load_required_constant, location)
@@ -395,7 +747,7 @@ module Prism
395
747
  source, load_node, load_location, load_node, location)
396
748
  when 40 then
397
749
  ConstantPathTargetNode.new(
398
- source, load_optional_node, load_node, load_location, location)
750
+ source, load_optional_node, load_optional_constant, load_location, load_location, location)
399
751
  when 41 then
400
752
  ConstantPathWriteNode.new(
401
753
  source, load_node, load_location, load_node, location)
@@ -409,7 +761,7 @@ module Prism
409
761
  ConstantWriteNode.new(
410
762
  source, load_required_constant, load_location, load_node, load_location, location)
411
763
  when 45 then
412
- load_serialized_length
764
+ load_uint32
413
765
  DefNode.new(
414
766
  source, load_required_constant, load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varuint) { load_required_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
415
767
  when 46 then
@@ -438,7 +790,7 @@ module Prism
438
790
  source, load_varuint, load_optional_node, load_optional_node, load_location, location)
439
791
  when 54 then
440
792
  FloatNode.new(
441
- source, location)
793
+ source, load_double, location)
442
794
  when 55 then
443
795
  ForNode.new(
444
796
  source, load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
@@ -522,7 +874,7 @@ module Prism
522
874
  source, load_required_constant, load_location, load_node, load_location, location)
523
875
  when 82 then
524
876
  IntegerNode.new(
525
- source, load_varuint, location)
877
+ source, load_varuint, load_integer, location)
526
878
  when 83 then
527
879
  InterpolatedMatchLastLineNode.new(
528
880
  source, load_varuint, load_location, Array.new(load_varuint) { load_node }, load_location, location)
@@ -531,7 +883,7 @@ module Prism
531
883
  source, load_varuint, load_location, Array.new(load_varuint) { load_node }, load_location, location)
532
884
  when 85 then
533
885
  InterpolatedStringNode.new(
534
- source, load_optional_location, Array.new(load_varuint) { load_node }, load_optional_location, location)
886
+ source, load_varuint, load_optional_location, Array.new(load_varuint) { load_node }, load_optional_location, location)
535
887
  when 86 then
536
888
  InterpolatedSymbolNode.new(
537
889
  source, load_optional_location, Array.new(load_varuint) { load_node }, load_optional_location, location)
@@ -539,186 +891,192 @@ module Prism
539
891
  InterpolatedXStringNode.new(
540
892
  source, load_location, Array.new(load_varuint) { load_node }, load_location, location)
541
893
  when 88 then
894
+ ItParametersNode.new(
895
+ source, location)
896
+ when 89 then
542
897
  KeywordHashNode.new(
543
898
  source, load_varuint, Array.new(load_varuint) { load_node }, location)
544
- when 89 then
899
+ when 90 then
545
900
  KeywordRestParameterNode.new(
546
901
  source, load_varuint, load_optional_constant, load_optional_location, load_location, location)
547
- when 90 then
902
+ when 91 then
548
903
  LambdaNode.new(
549
904
  source, Array.new(load_varuint) { load_required_constant }, load_location, load_location, load_location, load_optional_node, load_optional_node, location)
550
- when 91 then
905
+ when 92 then
551
906
  LocalVariableAndWriteNode.new(
552
907
  source, load_location, load_location, load_node, load_required_constant, load_varuint, location)
553
- when 92 then
908
+ when 93 then
554
909
  LocalVariableOperatorWriteNode.new(
555
910
  source, load_location, load_location, load_node, load_required_constant, load_required_constant, load_varuint, location)
556
- when 93 then
911
+ when 94 then
557
912
  LocalVariableOrWriteNode.new(
558
913
  source, load_location, load_location, load_node, load_required_constant, load_varuint, location)
559
- when 94 then
914
+ when 95 then
560
915
  LocalVariableReadNode.new(
561
916
  source, load_required_constant, load_varuint, location)
562
- when 95 then
917
+ when 96 then
563
918
  LocalVariableTargetNode.new(
564
919
  source, load_required_constant, load_varuint, location)
565
- when 96 then
920
+ when 97 then
566
921
  LocalVariableWriteNode.new(
567
922
  source, load_required_constant, load_varuint, load_location, load_node, load_location, location)
568
- when 97 then
923
+ when 98 then
569
924
  MatchLastLineNode.new(
570
925
  source, load_varuint, load_location, load_location, load_location, load_string, location)
571
- when 98 then
926
+ when 99 then
572
927
  MatchPredicateNode.new(
573
928
  source, load_node, load_node, load_location, location)
574
- when 99 then
929
+ when 100 then
575
930
  MatchRequiredNode.new(
576
931
  source, load_node, load_node, load_location, location)
577
- when 100 then
932
+ when 101 then
578
933
  MatchWriteNode.new(
579
934
  source, load_node, Array.new(load_varuint) { load_node }, location)
580
- when 101 then
935
+ when 102 then
581
936
  MissingNode.new(
582
937
  source, location)
583
- when 102 then
938
+ when 103 then
584
939
  ModuleNode.new(
585
940
  source, Array.new(load_varuint) { load_required_constant }, load_location, load_node, load_optional_node, load_location, load_required_constant, location)
586
- when 103 then
941
+ when 104 then
587
942
  MultiTargetNode.new(
588
943
  source, Array.new(load_varuint) { load_node }, load_optional_node, Array.new(load_varuint) { load_node }, load_optional_location, load_optional_location, location)
589
- when 104 then
944
+ when 105 then
590
945
  MultiWriteNode.new(
591
946
  source, Array.new(load_varuint) { load_node }, load_optional_node, Array.new(load_varuint) { load_node }, load_optional_location, load_optional_location, load_location, load_node, location)
592
- when 105 then
947
+ when 106 then
593
948
  NextNode.new(
594
949
  source, load_optional_node, load_location, location)
595
- when 106 then
950
+ when 107 then
596
951
  NilNode.new(
597
952
  source, location)
598
- when 107 then
953
+ when 108 then
599
954
  NoKeywordsParameterNode.new(
600
955
  source, load_location, load_location, location)
601
- when 108 then
956
+ when 109 then
602
957
  NumberedParametersNode.new(
603
958
  source, io.getbyte, location)
604
- when 109 then
959
+ when 110 then
605
960
  NumberedReferenceReadNode.new(
606
961
  source, load_varuint, location)
607
- when 110 then
962
+ when 111 then
608
963
  OptionalKeywordParameterNode.new(
609
964
  source, load_varuint, load_required_constant, load_location, load_node, location)
610
- when 111 then
965
+ when 112 then
611
966
  OptionalParameterNode.new(
612
967
  source, load_varuint, load_required_constant, load_location, load_location, load_node, location)
613
- when 112 then
968
+ when 113 then
614
969
  OrNode.new(
615
970
  source, load_node, load_node, load_location, location)
616
- when 113 then
971
+ when 114 then
617
972
  ParametersNode.new(
618
973
  source, Array.new(load_varuint) { load_node }, Array.new(load_varuint) { load_node }, load_optional_node, Array.new(load_varuint) { load_node }, Array.new(load_varuint) { load_node }, load_optional_node, load_optional_node, location)
619
- when 114 then
974
+ when 115 then
620
975
  ParenthesesNode.new(
621
976
  source, load_optional_node, load_location, load_location, location)
622
- when 115 then
977
+ when 116 then
623
978
  PinnedExpressionNode.new(
624
979
  source, load_node, load_location, load_location, load_location, location)
625
- when 116 then
980
+ when 117 then
626
981
  PinnedVariableNode.new(
627
982
  source, load_node, load_location, location)
628
- when 117 then
983
+ when 118 then
629
984
  PostExecutionNode.new(
630
985
  source, load_optional_node, load_location, load_location, load_location, location)
631
- when 118 then
986
+ when 119 then
632
987
  PreExecutionNode.new(
633
988
  source, load_optional_node, load_location, load_location, load_location, location)
634
- when 119 then
989
+ when 120 then
635
990
  ProgramNode.new(
636
991
  source, Array.new(load_varuint) { load_required_constant }, load_node, location)
637
- when 120 then
992
+ when 121 then
638
993
  RangeNode.new(
639
994
  source, load_varuint, load_optional_node, load_optional_node, load_location, location)
640
- when 121 then
995
+ when 122 then
641
996
  RationalNode.new(
642
997
  source, load_node, location)
643
- when 122 then
998
+ when 123 then
644
999
  RedoNode.new(
645
1000
  source, location)
646
- when 123 then
1001
+ when 124 then
647
1002
  RegularExpressionNode.new(
648
1003
  source, load_varuint, load_location, load_location, load_location, load_string, location)
649
- when 124 then
1004
+ when 125 then
650
1005
  RequiredKeywordParameterNode.new(
651
1006
  source, load_varuint, load_required_constant, load_location, location)
652
- when 125 then
1007
+ when 126 then
653
1008
  RequiredParameterNode.new(
654
1009
  source, load_varuint, load_required_constant, location)
655
- when 126 then
1010
+ when 127 then
656
1011
  RescueModifierNode.new(
657
1012
  source, load_node, load_location, load_node, location)
658
- when 127 then
1013
+ when 128 then
659
1014
  RescueNode.new(
660
1015
  source, load_location, Array.new(load_varuint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
661
- when 128 then
1016
+ when 129 then
662
1017
  RestParameterNode.new(
663
1018
  source, load_varuint, load_optional_constant, load_optional_location, load_location, location)
664
- when 129 then
1019
+ when 130 then
665
1020
  RetryNode.new(
666
1021
  source, location)
667
- when 130 then
668
- ReturnNode.new(
669
- source, load_location, load_optional_node, location)
670
1022
  when 131 then
1023
+ ReturnNode.new(
1024
+ source, load_varuint, load_location, load_optional_node, location)
1025
+ when 132 then
671
1026
  SelfNode.new(
672
1027
  source, location)
673
- when 132 then
1028
+ when 133 then
1029
+ ShareableConstantNode.new(
1030
+ source, load_varuint, load_node, location)
1031
+ when 134 then
674
1032
  SingletonClassNode.new(
675
1033
  source, Array.new(load_varuint) { load_required_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
676
- when 133 then
1034
+ when 135 then
677
1035
  SourceEncodingNode.new(
678
1036
  source, location)
679
- when 134 then
1037
+ when 136 then
680
1038
  SourceFileNode.new(
681
- source, load_string, location)
682
- when 135 then
1039
+ source, load_varuint, load_string, location)
1040
+ when 137 then
683
1041
  SourceLineNode.new(
684
1042
  source, location)
685
- when 136 then
1043
+ when 138 then
686
1044
  SplatNode.new(
687
1045
  source, load_location, load_optional_node, location)
688
- when 137 then
1046
+ when 139 then
689
1047
  StatementsNode.new(
690
1048
  source, Array.new(load_varuint) { load_node }, location)
691
- when 138 then
1049
+ when 140 then
692
1050
  StringNode.new(
693
1051
  source, load_varuint, load_optional_location, load_location, load_optional_location, load_string, location)
694
- when 139 then
1052
+ when 141 then
695
1053
  SuperNode.new(
696
1054
  source, load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
697
- when 140 then
1055
+ when 142 then
698
1056
  SymbolNode.new(
699
1057
  source, load_varuint, load_optional_location, load_optional_location, load_optional_location, load_string, location)
700
- when 141 then
1058
+ when 143 then
701
1059
  TrueNode.new(
702
1060
  source, location)
703
- when 142 then
1061
+ when 144 then
704
1062
  UndefNode.new(
705
1063
  source, Array.new(load_varuint) { load_node }, load_location, location)
706
- when 143 then
1064
+ when 145 then
707
1065
  UnlessNode.new(
708
1066
  source, load_location, load_node, load_optional_location, load_optional_node, load_optional_node, load_optional_location, location)
709
- when 144 then
1067
+ when 146 then
710
1068
  UntilNode.new(
711
1069
  source, load_varuint, load_location, load_optional_location, load_node, load_optional_node, location)
712
- when 145 then
1070
+ when 147 then
713
1071
  WhenNode.new(
714
- source, load_location, Array.new(load_varuint) { load_node }, load_optional_node, location)
715
- when 146 then
1072
+ source, load_location, Array.new(load_varuint) { load_node }, load_optional_location, load_optional_node, location)
1073
+ when 148 then
716
1074
  WhileNode.new(
717
1075
  source, load_varuint, load_location, load_optional_location, load_node, load_optional_node, location)
718
- when 147 then
1076
+ when 149 then
719
1077
  XStringNode.new(
720
1078
  source, load_varuint, load_location, load_location, load_location, load_string, location)
721
- when 148 then
1079
+ when 150 then
722
1080
  YieldNode.new(
723
1081
  source, load_location, load_optional_location, load_optional_node, load_optional_location, location)
724
1082
  end
@@ -890,7 +1248,7 @@ module Prism
890
1248
  -> {
891
1249
  location = load_location
892
1250
  ClassVariableWriteNode.new(
893
- source, load_required_constant, load_location, load_node, load_optional_location, location)
1251
+ source, load_required_constant, load_location, load_node, load_location, location)
894
1252
  },
895
1253
  -> {
896
1254
  location = load_location
@@ -915,7 +1273,7 @@ module Prism
915
1273
  -> {
916
1274
  location = load_location
917
1275
  ConstantPathNode.new(
918
- source, load_optional_node, load_node, load_location, location)
1276
+ source, load_optional_node, load_optional_constant, load_location, load_location, location)
919
1277
  },
920
1278
  -> {
921
1279
  location = load_location
@@ -930,7 +1288,7 @@ module Prism
930
1288
  -> {
931
1289
  location = load_location
932
1290
  ConstantPathTargetNode.new(
933
- source, load_optional_node, load_node, load_location, location)
1291
+ source, load_optional_node, load_optional_constant, load_location, load_location, location)
934
1292
  },
935
1293
  -> {
936
1294
  location = load_location
@@ -954,7 +1312,7 @@ module Prism
954
1312
  },
955
1313
  -> {
956
1314
  location = load_location
957
- load_serialized_length
1315
+ load_uint32
958
1316
  DefNode.new(
959
1317
  source, load_required_constant, load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varuint) { load_required_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
960
1318
  },
@@ -1001,7 +1359,7 @@ module Prism
1001
1359
  -> {
1002
1360
  location = load_location
1003
1361
  FloatNode.new(
1004
- source, location)
1362
+ source, load_double, location)
1005
1363
  },
1006
1364
  -> {
1007
1365
  location = load_location
@@ -1141,7 +1499,7 @@ module Prism
1141
1499
  -> {
1142
1500
  location = load_location
1143
1501
  IntegerNode.new(
1144
- source, load_varuint, location)
1502
+ source, load_varuint, load_integer, location)
1145
1503
  },
1146
1504
  -> {
1147
1505
  location = load_location
@@ -1156,7 +1514,7 @@ module Prism
1156
1514
  -> {
1157
1515
  location = load_location
1158
1516
  InterpolatedStringNode.new(
1159
- source, load_optional_location, Array.new(load_varuint) { load_node }, load_optional_location, location)
1517
+ source, load_varuint, load_optional_location, Array.new(load_varuint) { load_node }, load_optional_location, location)
1160
1518
  },
1161
1519
  -> {
1162
1520
  location = load_location
@@ -1168,6 +1526,11 @@ module Prism
1168
1526
  InterpolatedXStringNode.new(
1169
1527
  source, load_location, Array.new(load_varuint) { load_node }, load_location, location)
1170
1528
  },
1529
+ -> {
1530
+ location = load_location
1531
+ ItParametersNode.new(
1532
+ source, location)
1533
+ },
1171
1534
  -> {
1172
1535
  location = load_location
1173
1536
  KeywordHashNode.new(
@@ -1381,13 +1744,18 @@ module Prism
1381
1744
  -> {
1382
1745
  location = load_location
1383
1746
  ReturnNode.new(
1384
- source, load_location, load_optional_node, location)
1747
+ source, load_varuint, load_location, load_optional_node, location)
1385
1748
  },
1386
1749
  -> {
1387
1750
  location = load_location
1388
1751
  SelfNode.new(
1389
1752
  source, location)
1390
1753
  },
1754
+ -> {
1755
+ location = load_location
1756
+ ShareableConstantNode.new(
1757
+ source, load_varuint, load_node, location)
1758
+ },
1391
1759
  -> {
1392
1760
  location = load_location
1393
1761
  SingletonClassNode.new(
@@ -1401,7 +1769,7 @@ module Prism
1401
1769
  -> {
1402
1770
  location = load_location
1403
1771
  SourceFileNode.new(
1404
- source, load_string, location)
1772
+ source, load_varuint, load_string, location)
1405
1773
  },
1406
1774
  -> {
1407
1775
  location = load_location
@@ -1456,7 +1824,7 @@ module Prism
1456
1824
  -> {
1457
1825
  location = load_location
1458
1826
  WhenNode.new(
1459
- source, load_location, Array.new(load_varuint) { load_node }, load_optional_node, location)
1827
+ source, load_location, Array.new(load_varuint) { load_node }, load_optional_location, load_optional_node, location)
1460
1828
  },
1461
1829
  -> {
1462
1830
  location = load_location