syntax_tree 3.6.3 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,29 +60,46 @@ module SyntaxTree
60
60
  # This represents all of the tokens coming back from the lexer. It is
61
61
  # replacing a simple array because it keeps track of the last deleted token
62
62
  # from the list for better error messages.
63
- class TokenList < SimpleDelegator
64
- attr_reader :last_deleted
63
+ class TokenList
64
+ attr_reader :tokens, :last_deleted
65
65
 
66
- def initialize(object)
67
- super
66
+ def initialize
67
+ @tokens = []
68
68
  @last_deleted = nil
69
69
  end
70
70
 
71
+ def <<(token)
72
+ tokens << token
73
+ end
74
+
75
+ def [](index)
76
+ tokens[index]
77
+ end
78
+
79
+ def any?(&block)
80
+ tokens.any?(&block)
81
+ end
82
+
83
+ def reverse_each(&block)
84
+ tokens.reverse_each(&block)
85
+ end
86
+
87
+ def rindex(&block)
88
+ tokens.rindex(&block)
89
+ end
90
+
71
91
  def delete(value)
72
- @last_deleted = super || @last_deleted
92
+ @last_deleted = tokens.delete(value) || @last_deleted
73
93
  end
74
94
 
75
95
  def delete_at(index)
76
- @last_deleted = super
96
+ @last_deleted = tokens.delete_at(index)
77
97
  end
78
98
  end
79
99
 
80
100
  # [String] the source being parsed
81
101
  attr_reader :source
82
102
 
83
- # [Array[ String ]] the list of lines in the source
84
- attr_reader :lines
85
-
86
103
  # [Array[ SingleByteString | MultiByteString ]] the list of objects that
87
104
  # represent the start of each line in character offsets
88
105
  attr_reader :line_counts
@@ -105,12 +122,6 @@ module SyntaxTree
105
122
  # example.
106
123
  @source = source
107
124
 
108
- # Similarly, we keep the lines of the source string around to be able to
109
- # check if certain lines contain certain characters. For example, we'll
110
- # use this to generate the content that goes after the __END__ keyword.
111
- # Or we'll use this to check if a comment has other content on its line.
112
- @lines = source.split(/\r?\n/)
113
-
114
125
  # This is the full set of comments that have been found by the parser.
115
126
  # It's a running list. At the end of every block of statements, they will
116
127
  # go in and attempt to grab any comments that are on their own line and
@@ -144,7 +155,7 @@ module SyntaxTree
144
155
  # Most of the time, when a parser event consumes one of these events, it
145
156
  # will be deleted from the list. So ideally, this list stays pretty short
146
157
  # over the course of parsing a source string.
147
- @tokens = TokenList.new([])
158
+ @tokens = TokenList.new
148
159
 
149
160
  # Here we're going to build up a list of SingleByteString or
150
161
  # MultiByteString objects. They're each going to represent a string in the
@@ -153,7 +164,7 @@ module SyntaxTree
153
164
  @line_counts = []
154
165
  last_index = 0
155
166
 
156
- @source.lines.each do |line|
167
+ @source.each_line do |line|
157
168
  @line_counts << if line.size == line.bytesize
158
169
  SingleByteString.new(last_index)
159
170
  else
@@ -233,28 +244,55 @@ module SyntaxTree
233
244
  # "module" (which would happen to be the innermost keyword). Then the outer
234
245
  # one would only be able to grab the first one. In this way all of the
235
246
  # tokens act as their own stack.
236
- def find_token(type, value = :any, consume: true, location: nil)
237
- index =
238
- tokens.rindex do |token|
239
- token.is_a?(type) && (value == :any || (token.value == value))
240
- end
247
+ #
248
+ # If we're expecting to be able to find a token and consume it, but can't
249
+ # actually find it, then we need to raise an error. This is _usually_ caused
250
+ # by a syntax error in the source that we're printing. It could also be
251
+ # caused by accidentally attempting to consume a token twice by two
252
+ # different parser event handlers.
241
253
 
242
- if consume
243
- # If we're expecting to be able to find a token and consume it, but
244
- # can't actually find it, then we need to raise an error. This is
245
- # _usually_ caused by a syntax error in the source that we're printing.
246
- # It could also be caused by accidentally attempting to consume a token
247
- # twice by two different parser event handlers.
248
- unless index
249
- token = value == :any ? type.name.split("::", 2).last : value
250
- message = "Cannot find expected #{token}"
251
- raise ParseError.new(message, *find_token_error(location))
252
- end
254
+ def find_token(type)
255
+ index = tokens.rindex { |token| token.is_a?(type) }
256
+ tokens[index] if index
257
+ end
253
258
 
254
- tokens.delete_at(index)
255
- elsif index
256
- tokens[index]
257
- end
259
+ def find_keyword(name)
260
+ index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) }
261
+ tokens[index] if index
262
+ end
263
+
264
+ def find_operator(name)
265
+ index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) }
266
+ tokens[index] if index
267
+ end
268
+
269
+ def consume_error(name, location)
270
+ message = "Cannot find expected #{name}"
271
+ raise ParseError.new(message, *find_token_error(location))
272
+ end
273
+
274
+ def consume_token(type)
275
+ index = tokens.rindex { |token| token.is_a?(type) }
276
+ consume_error(type.name.split("::", 2).last, nil) unless index
277
+ tokens.delete_at(index)
278
+ end
279
+
280
+ def consume_tstring_end(location)
281
+ index = tokens.rindex { |token| token.is_a?(TStringEnd) }
282
+ consume_error("string ending", location) unless index
283
+ tokens.delete_at(index)
284
+ end
285
+
286
+ def consume_keyword(name)
287
+ index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) }
288
+ consume_error(name, nil) unless index
289
+ tokens.delete_at(index)
290
+ end
291
+
292
+ def consume_operator(name)
293
+ index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) }
294
+ consume_error(name, nil) unless index
295
+ tokens.delete_at(index)
258
296
  end
259
297
 
260
298
  # A helper function to find a :: operator. We do special handling instead of
@@ -283,13 +321,18 @@ module SyntaxTree
283
321
  # By finding the next non-space character, we can make sure that the bounds
284
322
  # of the statement list are correct.
285
323
  def find_next_statement_start(position)
286
- remaining = source[position..]
287
-
288
- if remaining.sub(/\A +/, "")[0] == "#"
289
- return position + remaining.index("\n")
324
+ maximum = source.length
325
+
326
+ position.upto(maximum) do |pound_index|
327
+ case source[pound_index]
328
+ when "#"
329
+ return source.index("\n", pound_index + 1) || maximum
330
+ when " "
331
+ # continue
332
+ else
333
+ return position
334
+ end
290
335
  end
291
-
292
- position
293
336
  end
294
337
 
295
338
  # --------------------------------------------------------------------------
@@ -300,8 +343,8 @@ module SyntaxTree
300
343
  # :call-seq:
301
344
  # on_BEGIN: (Statements statements) -> BEGINBlock
302
345
  def on_BEGIN(statements)
303
- lbrace = find_token(LBrace)
304
- rbrace = find_token(RBrace)
346
+ lbrace = consume_token(LBrace)
347
+ rbrace = consume_token(RBrace)
305
348
 
306
349
  start_char = find_next_statement_start(lbrace.location.end_char)
307
350
  statements.bind(
@@ -311,7 +354,7 @@ module SyntaxTree
311
354
  rbrace.location.start_column
312
355
  )
313
356
 
314
- keyword = find_token(Kw, "BEGIN")
357
+ keyword = consume_keyword(:BEGIN)
315
358
 
316
359
  BEGINBlock.new(
317
360
  lbrace: lbrace,
@@ -338,8 +381,8 @@ module SyntaxTree
338
381
  # :call-seq:
339
382
  # on_END: (Statements statements) -> ENDBlock
340
383
  def on_END(statements)
341
- lbrace = find_token(LBrace)
342
- rbrace = find_token(RBrace)
384
+ lbrace = consume_token(LBrace)
385
+ rbrace = consume_token(RBrace)
343
386
 
344
387
  start_char = find_next_statement_start(lbrace.location.end_char)
345
388
  statements.bind(
@@ -349,7 +392,7 @@ module SyntaxTree
349
392
  rbrace.location.start_column
350
393
  )
351
394
 
352
- keyword = find_token(Kw, "END")
395
+ keyword = consume_keyword(:END)
353
396
 
354
397
  ENDBlock.new(
355
398
  lbrace: lbrace,
@@ -380,7 +423,7 @@ module SyntaxTree
380
423
  # (DynaSymbol | SymbolLiteral) right
381
424
  # ) -> Alias
382
425
  def on_alias(left, right)
383
- keyword = find_token(Kw, "alias")
426
+ keyword = consume_keyword(:alias)
384
427
 
385
428
  Alias.new(
386
429
  left: left,
@@ -392,8 +435,8 @@ module SyntaxTree
392
435
  # :call-seq:
393
436
  # on_aref: (untyped collection, (nil | Args) index) -> ARef
394
437
  def on_aref(collection, index)
395
- find_token(LBracket)
396
- rbracket = find_token(RBracket)
438
+ consume_token(LBracket)
439
+ rbracket = consume_token(RBracket)
397
440
 
398
441
  ARef.new(
399
442
  collection: collection,
@@ -408,8 +451,8 @@ module SyntaxTree
408
451
  # (nil | Args) index
409
452
  # ) -> ARefField
410
453
  def on_aref_field(collection, index)
411
- find_token(LBracket)
412
- rbracket = find_token(RBracket)
454
+ consume_token(LBracket)
455
+ rbracket = consume_token(RBracket)
413
456
 
414
457
  ARefField.new(
415
458
  collection: collection,
@@ -427,8 +470,8 @@ module SyntaxTree
427
470
  # (nil | Args | ArgsForward) arguments
428
471
  # ) -> ArgParen
429
472
  def on_arg_paren(arguments)
430
- lparen = find_token(LParen)
431
- rparen = find_token(RParen)
473
+ lparen = consume_token(LParen)
474
+ rparen = consume_token(RParen)
432
475
 
433
476
  # If the arguments exceed the ending of the parentheses, then we know we
434
477
  # have a heredoc in the arguments, and we need to use the bounds of the
@@ -470,23 +513,26 @@ module SyntaxTree
470
513
  # (false | untyped) block
471
514
  # ) -> Args
472
515
  def on_args_add_block(arguments, block)
516
+ end_char = arguments.parts.any? && arguments.location.end_char
517
+
473
518
  # First, see if there is an & operator that could potentially be
474
519
  # associated with the block part of this args_add_block. If there is not,
475
520
  # then just return the arguments.
476
- operator = find_token(Op, "&", consume: false)
477
- return arguments unless operator
478
-
479
- # If there are any arguments and the operator we found from the list is
480
- # not after them, then we're going to return the arguments as-is because
481
- # we're looking at an & that occurs before the arguments are done.
482
- if arguments.parts.any? &&
483
- operator.location.start_char < arguments.location.end_char
484
- return arguments
485
- end
521
+ index =
522
+ tokens.rindex do |token|
523
+ # If there are any arguments and the operator we found from the list
524
+ # is not after them, then we're going to return the arguments as-is
525
+ # because we're looking at an & that occurs before the arguments are
526
+ # done.
527
+ return arguments if end_char && token.location.start_char < end_char
528
+ token.is_a?(Op) && (token.name == :&)
529
+ end
530
+
531
+ return arguments unless index
486
532
 
487
533
  # Now we know we have an & operator, so we're going to delete it from the
488
534
  # list of tokens to make sure it doesn't get confused with anything else.
489
- tokens.delete(operator)
535
+ operator = tokens.delete_at(index)
490
536
 
491
537
  # Construct the location that represents the block argument.
492
538
  location = operator.location
@@ -505,7 +551,7 @@ module SyntaxTree
505
551
  # :call-seq:
506
552
  # on_args_add_star: (Args arguments, untyped star) -> Args
507
553
  def on_args_add_star(arguments, argument)
508
- beginning = find_token(Op, "*")
554
+ beginning = consume_operator(:*)
509
555
  ending = argument || beginning
510
556
 
511
557
  location =
@@ -527,7 +573,7 @@ module SyntaxTree
527
573
  # :call-seq:
528
574
  # on_args_forward: () -> ArgsForward
529
575
  def on_args_forward
530
- op = find_token(Op, "...")
576
+ op = consume_operator(:"...")
531
577
 
532
578
  ArgsForward.new(value: op.value, location: op.location)
533
579
  end
@@ -547,8 +593,8 @@ module SyntaxTree
547
593
  # ArrayLiteral | QSymbols | QWords | Symbols | Words
548
594
  def on_array(contents)
549
595
  if !contents || contents.is_a?(Args)
550
- lbracket = find_token(LBracket)
551
- rbracket = find_token(RBracket)
596
+ lbracket = consume_token(LBracket)
597
+ rbracket = consume_token(RBracket)
552
598
 
553
599
  ArrayLiteral.new(
554
600
  lbracket: lbracket,
@@ -556,8 +602,7 @@ module SyntaxTree
556
602
  location: lbracket.location.to(rbracket.location)
557
603
  )
558
604
  else
559
- tstring_end =
560
- find_token(TStringEnd, location: contents.beginning.location)
605
+ tstring_end = consume_tstring_end(contents.beginning.location)
561
606
 
562
607
  contents.class.new(
563
608
  beginning: contents.beginning,
@@ -567,6 +612,56 @@ module SyntaxTree
567
612
  end
568
613
  end
569
614
 
615
+ # Ugh... I really do not like this class. Basically, ripper doesn't provide
616
+ # enough information about where pins are located in the tree. It only gives
617
+ # events for ^ ops and var_ref nodes. You have to piece it together
618
+ # yourself.
619
+ #
620
+ # Note that there are edge cases here that we straight up do not address,
621
+ # because I honestly think it's going to be faster to write a new parser
622
+ # than to address them. For example, this will not work properly:
623
+ #
624
+ # foo in ^((bar = 0; bar; baz))
625
+ #
626
+ # If someone actually does something like that, we'll have to find another
627
+ # way to make this work.
628
+ class PinVisitor < Visitor
629
+ attr_reader :pins, :stack
630
+
631
+ def initialize(pins)
632
+ @pins = pins
633
+ @stack = []
634
+ end
635
+
636
+ def visit(node)
637
+ return if pins.empty?
638
+ stack << node
639
+ super
640
+ stack.pop
641
+ end
642
+
643
+ def visit_var_ref(node)
644
+ pins.shift
645
+ node.pin(stack[-2])
646
+ end
647
+
648
+ def self.visit(node, tokens)
649
+ start_char = node.location.start_char
650
+ allocated = []
651
+
652
+ tokens.reverse_each do |token|
653
+ char = token.location.start_char
654
+ break if char <= start_char
655
+
656
+ if token.is_a?(Op) && token.value == "^"
657
+ allocated.unshift(tokens.delete(token))
658
+ end
659
+ end
660
+
661
+ new(allocated).visit(node) if allocated.any?
662
+ end
663
+ end
664
+
570
665
  # :call-seq:
571
666
  # on_aryptn: (
572
667
  # (nil | VarRef) constant,
@@ -583,7 +678,7 @@ module SyntaxTree
583
678
  # of the various parts.
584
679
  location =
585
680
  if parts.empty?
586
- find_token(LBracket).location.to(find_token(RBracket).location)
681
+ consume_token(LBracket).location.to(consume_token(RBracket).location)
587
682
  else
588
683
  parts[0].location.to(parts[-1].location)
589
684
  end
@@ -594,12 +689,13 @@ module SyntaxTree
594
689
  if rest.is_a?(VarField) && rest.value.nil?
595
690
  tokens.rindex do |rtoken|
596
691
  case rtoken
597
- in Op[value: "*"]
598
- rest = VarField.new(value: nil, location: rtoken.location)
692
+ when Comma
599
693
  break
600
- in Comma
601
- break
602
- else
694
+ when Op
695
+ if rtoken.value == "*"
696
+ rest = VarField.new(value: nil, location: rtoken.location)
697
+ break
698
+ end
603
699
  end
604
700
  end
605
701
  end
@@ -644,7 +740,7 @@ module SyntaxTree
644
740
  # :call-seq:
645
741
  # on_assoc_splat: (untyped value) -> AssocSplat
646
742
  def on_assoc_splat(value)
647
- operator = find_token(Op, "**")
743
+ operator = consume_operator(:**)
648
744
 
649
745
  AssocSplat.new(
650
746
  value: value,
@@ -704,23 +800,23 @@ module SyntaxTree
704
800
  # :call-seq:
705
801
  # on_begin: (untyped bodystmt) -> Begin | PinnedBegin
706
802
  def on_begin(bodystmt)
707
- pin = find_token(Op, "^", consume: false)
803
+ pin = find_operator(:^)
708
804
 
709
805
  if pin && pin.location.start_char < bodystmt.location.start_char
710
806
  tokens.delete(pin)
711
- find_token(LParen)
807
+ consume_token(LParen)
712
808
 
713
- rparen = find_token(RParen)
809
+ rparen = consume_token(RParen)
714
810
  location = pin.location.to(rparen.location)
715
811
 
716
812
  PinnedBegin.new(statement: bodystmt, location: location)
717
813
  else
718
- keyword = find_token(Kw, "begin")
814
+ keyword = consume_keyword(:begin)
719
815
  end_location =
720
816
  if bodystmt.else_clause
721
817
  bodystmt.location
722
818
  else
723
- find_token(Kw, "end").location
819
+ consume_keyword(:end).location
724
820
  end
725
821
 
726
822
  bodystmt.bind(
@@ -746,13 +842,11 @@ module SyntaxTree
746
842
  # Here, we're going to search backward for the token that's between the
747
843
  # two operands that matches the operator so we can delete it from the
748
844
  # list.
845
+ range = (left.location.end_char + 1)...right.location.start_char
749
846
  index =
750
847
  tokens.rindex do |token|
751
- location = token.location
752
-
753
- token.is_a?(Op) && token.value == operator.to_s &&
754
- location.start_char > left.location.end_char &&
755
- location.end_char < right.location.start_char
848
+ token.is_a?(Op) && token.name == operator &&
849
+ range.cover?(token.location.start_char)
756
850
  end
757
851
 
758
852
  tokens.delete_at(index) if index
@@ -795,7 +889,7 @@ module SyntaxTree
795
889
  # :call-seq:
796
890
  # on_blockarg: (Ident name) -> BlockArg
797
891
  def on_blockarg(name)
798
- operator = find_token(Op, "&")
892
+ operator = consume_operator(:&)
799
893
 
800
894
  location = operator.location
801
895
  location = location.to(name.location) if name
@@ -814,7 +908,7 @@ module SyntaxTree
814
908
  BodyStmt.new(
815
909
  statements: statements,
816
910
  rescue_clause: rescue_clause,
817
- else_keyword: else_clause && find_token(Kw, "else"),
911
+ else_keyword: else_clause && consume_keyword(:else),
818
912
  else_clause: else_clause,
819
913
  ensure_clause: ensure_clause,
820
914
  location:
@@ -828,8 +922,8 @@ module SyntaxTree
828
922
  # Statements statements
829
923
  # ) -> BraceBlock
830
924
  def on_brace_block(block_var, statements)
831
- lbrace = find_token(LBrace)
832
- rbrace = find_token(RBrace)
925
+ lbrace = consume_token(LBrace)
926
+ rbrace = consume_token(RBrace)
833
927
  location = (block_var || lbrace).location
834
928
 
835
929
  start_char = find_next_statement_start(location.end_char)
@@ -864,7 +958,7 @@ module SyntaxTree
864
958
  # :call-seq:
865
959
  # on_break: (Args arguments) -> Break
866
960
  def on_break(arguments)
867
- keyword = find_token(Kw, "break")
961
+ keyword = consume_keyword(:break)
868
962
 
869
963
  location = keyword.location
870
964
  location = location.to(arguments.location) if arguments.parts.any?
@@ -900,7 +994,7 @@ module SyntaxTree
900
994
  # :call-seq:
901
995
  # on_case: (untyped value, untyped consequent) -> Case | RAssign
902
996
  def on_case(value, consequent)
903
- if (keyword = find_token(Kw, "case", consume: false))
997
+ if (keyword = find_keyword(:case))
904
998
  tokens.delete(keyword)
905
999
 
906
1000
  Case.new(
@@ -911,18 +1005,22 @@ module SyntaxTree
911
1005
  )
912
1006
  else
913
1007
  operator =
914
- if (keyword = find_token(Kw, "in", consume: false))
1008
+ if (keyword = find_keyword(:in))
915
1009
  tokens.delete(keyword)
916
1010
  else
917
- find_token(Op, "=>")
1011
+ consume_operator(:"=>")
918
1012
  end
919
1013
 
920
- RAssign.new(
921
- value: value,
922
- operator: operator,
923
- pattern: consequent,
924
- location: value.location.to(consequent.location)
925
- )
1014
+ node =
1015
+ RAssign.new(
1016
+ value: value,
1017
+ operator: operator,
1018
+ pattern: consequent,
1019
+ location: value.location.to(consequent.location)
1020
+ )
1021
+
1022
+ PinVisitor.visit(node, tokens)
1023
+ node
926
1024
  end
927
1025
  end
928
1026
 
@@ -933,8 +1031,8 @@ module SyntaxTree
933
1031
  # BodyStmt bodystmt
934
1032
  # ) -> ClassDeclaration
935
1033
  def on_class(constant, superclass, bodystmt)
936
- beginning = find_token(Kw, "class")
937
- ending = find_token(Kw, "end")
1034
+ beginning = consume_keyword(:class)
1035
+ ending = consume_keyword(:end)
938
1036
  location = (superclass || constant).location
939
1037
  start_char = find_next_statement_start(location.end_char)
940
1038
 
@@ -1004,20 +1102,20 @@ module SyntaxTree
1004
1102
  # :call-seq:
1005
1103
  # on_comment: (String value) -> Comment
1006
1104
  def on_comment(value)
1007
- line = lineno
1008
- comment =
1009
- Comment.new(
1010
- value: value.chomp,
1011
- inline: value.strip != lines[line - 1].strip,
1012
- location:
1013
- Location.token(
1014
- line: line,
1015
- char: char_pos,
1016
- column: current_column,
1017
- size: value.size - 1
1018
- )
1105
+ char = char_pos
1106
+ location =
1107
+ Location.token(
1108
+ line: lineno,
1109
+ char: char,
1110
+ column: current_column,
1111
+ size: value.size - 1
1019
1112
  )
1020
1113
 
1114
+ index = source.rindex(/[^\t ]/, char - 1) if char != 0
1115
+ inline = index && (source[index] != "\n")
1116
+ comment =
1117
+ Comment.new(value: value.chomp, inline: inline, location: location)
1118
+
1021
1119
  @comments << comment
1022
1120
  comment
1023
1121
  end
@@ -1092,7 +1190,7 @@ module SyntaxTree
1092
1190
 
1093
1191
  # Find the beginning of the method definition, which works for single-line
1094
1192
  # and normal method definitions.
1095
- beginning = find_token(Kw, "def")
1193
+ beginning = consume_keyword(:def)
1096
1194
 
1097
1195
  # If there aren't any params then we need to correct the params node
1098
1196
  # location information
@@ -1112,7 +1210,7 @@ module SyntaxTree
1112
1210
  params = Params.new(location: location)
1113
1211
  end
1114
1212
 
1115
- ending = find_token(Kw, "end", consume: false)
1213
+ ending = find_keyword(:end)
1116
1214
 
1117
1215
  if ending
1118
1216
  tokens.delete(ending)
@@ -1150,13 +1248,13 @@ module SyntaxTree
1150
1248
  # :call-seq:
1151
1249
  # on_defined: (untyped value) -> Defined
1152
1250
  def on_defined(value)
1153
- beginning = find_token(Kw, "defined?")
1251
+ beginning = consume_keyword(:defined?)
1154
1252
  ending = value
1155
1253
 
1156
1254
  range = beginning.location.end_char...value.location.start_char
1157
1255
  if source[range].include?("(")
1158
- find_token(LParen)
1159
- ending = find_token(RParen)
1256
+ consume_token(LParen)
1257
+ ending = consume_token(RParen)
1160
1258
  end
1161
1259
 
1162
1260
  Defined.new(
@@ -1197,8 +1295,8 @@ module SyntaxTree
1197
1295
  params = Params.new(location: location)
1198
1296
  end
1199
1297
 
1200
- beginning = find_token(Kw, "def")
1201
- ending = find_token(Kw, "end", consume: false)
1298
+ beginning = consume_keyword(:def)
1299
+ ending = find_keyword(:end)
1202
1300
 
1203
1301
  if ending
1204
1302
  tokens.delete(ending)
@@ -1238,8 +1336,8 @@ module SyntaxTree
1238
1336
  # :call-seq:
1239
1337
  # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock
1240
1338
  def on_do_block(block_var, bodystmt)
1241
- beginning = find_token(Kw, "do")
1242
- ending = find_token(Kw, "end")
1339
+ beginning = consume_keyword(:do)
1340
+ ending = consume_keyword(:end)
1243
1341
  location = (block_var || beginning).location
1244
1342
  start_char = find_next_statement_start(location.end_char)
1245
1343
 
@@ -1261,7 +1359,7 @@ module SyntaxTree
1261
1359
  # :call-seq:
1262
1360
  # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2
1263
1361
  def on_dot2(left, right)
1264
- operator = find_token(Op, "..")
1362
+ operator = consume_operator(:"..")
1265
1363
 
1266
1364
  beginning = left || operator
1267
1365
  ending = right || operator
@@ -1276,7 +1374,7 @@ module SyntaxTree
1276
1374
  # :call-seq:
1277
1375
  # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3
1278
1376
  def on_dot3(left, right)
1279
- operator = find_token(Op, "...")
1377
+ operator = consume_operator(:"...")
1280
1378
 
1281
1379
  beginning = left || operator
1282
1380
  ending = right || operator
@@ -1291,10 +1389,10 @@ module SyntaxTree
1291
1389
  # :call-seq:
1292
1390
  # on_dyna_symbol: (StringContent string_content) -> DynaSymbol
1293
1391
  def on_dyna_symbol(string_content)
1294
- if find_token(SymBeg, consume: false)
1392
+ if (symbeg = find_token(SymBeg))
1295
1393
  # A normal dynamic symbol
1296
- symbeg = find_token(SymBeg)
1297
- tstring_end = find_token(TStringEnd, location: symbeg.location)
1394
+ tokens.delete(symbeg)
1395
+ tstring_end = consume_tstring_end(symbeg.location)
1298
1396
 
1299
1397
  DynaSymbol.new(
1300
1398
  quote: symbeg.value,
@@ -1303,8 +1401,8 @@ module SyntaxTree
1303
1401
  )
1304
1402
  else
1305
1403
  # A dynamic symbol as a hash key
1306
- tstring_beg = find_token(TStringBeg)
1307
- label_end = find_token(LabelEnd)
1404
+ tstring_beg = consume_token(TStringBeg)
1405
+ label_end = consume_token(LabelEnd)
1308
1406
 
1309
1407
  DynaSymbol.new(
1310
1408
  parts: string_content.parts,
@@ -1317,7 +1415,7 @@ module SyntaxTree
1317
1415
  # :call-seq:
1318
1416
  # on_else: (Statements statements) -> Else
1319
1417
  def on_else(statements)
1320
- keyword = find_token(Kw, "else")
1418
+ keyword = consume_keyword(:else)
1321
1419
 
1322
1420
  # else can either end with an end keyword (in which case we'll want to
1323
1421
  # consume that event) or it can end with an ensure keyword (in which case
@@ -1357,8 +1455,8 @@ module SyntaxTree
1357
1455
  # (nil | Elsif | Else) consequent
1358
1456
  # ) -> Elsif
1359
1457
  def on_elsif(predicate, statements, consequent)
1360
- beginning = find_token(Kw, "elsif")
1361
- ending = consequent || find_token(Kw, "end")
1458
+ beginning = consume_keyword(:elsif)
1459
+ ending = consequent || consume_keyword(:end)
1362
1460
 
1363
1461
  start_char = find_next_statement_start(predicate.location.end_char)
1364
1462
  statements.bind(
@@ -1478,11 +1576,11 @@ module SyntaxTree
1478
1576
  # :call-seq:
1479
1577
  # on_ensure: (Statements statements) -> Ensure
1480
1578
  def on_ensure(statements)
1481
- keyword = find_token(Kw, "ensure")
1579
+ keyword = consume_keyword(:ensure)
1482
1580
 
1483
1581
  # We don't want to consume the :@kw event, because that would break
1484
1582
  # def..ensure..end chains.
1485
- ending = find_token(Kw, "end", consume: false)
1583
+ ending = find_keyword(:end)
1486
1584
  start_char = find_next_statement_start(keyword.location.end_char)
1487
1585
  statements.bind(
1488
1586
  start_char,
@@ -1504,7 +1602,7 @@ module SyntaxTree
1504
1602
  # :call-seq:
1505
1603
  # on_excessed_comma: () -> ExcessedComma
1506
1604
  def on_excessed_comma(*)
1507
- comma = find_token(Comma)
1605
+ comma = consume_token(Comma)
1508
1606
 
1509
1607
  ExcessedComma.new(value: comma.value, location: comma.location)
1510
1608
  end
@@ -1557,20 +1655,18 @@ module SyntaxTree
1557
1655
  # right left parenthesis, or the left splat. We're going to use this to
1558
1656
  # determine how to find the closing of the pattern, as well as determining
1559
1657
  # the location of the node.
1560
- opening =
1561
- find_token(LBracket, consume: false) ||
1562
- find_token(LParen, consume: false) || left
1658
+ opening = find_token(LBracket) || find_token(LParen) || left
1563
1659
 
1564
1660
  # The closing is based on the opening, which is either the matched
1565
1661
  # punctuation or the right splat.
1566
1662
  closing =
1567
1663
  case opening
1568
- in LBracket
1664
+ when LBracket
1569
1665
  tokens.delete(opening)
1570
- find_token(RBracket)
1571
- in LParen
1666
+ consume_token(RBracket)
1667
+ when LParen
1572
1668
  tokens.delete(opening)
1573
- find_token(RParen)
1669
+ consume_token(RParen)
1574
1670
  else
1575
1671
  right
1576
1672
  end
@@ -1591,13 +1687,13 @@ module SyntaxTree
1591
1687
  # Statements statements
1592
1688
  # ) -> For
1593
1689
  def on_for(index, collection, statements)
1594
- beginning = find_token(Kw, "for")
1595
- in_keyword = find_token(Kw, "in")
1596
- ending = find_token(Kw, "end")
1690
+ beginning = consume_keyword(:for)
1691
+ in_keyword = consume_keyword(:in)
1692
+ ending = consume_keyword(:end)
1597
1693
 
1598
1694
  # Consume the do keyword if it exists so that it doesn't get confused for
1599
1695
  # some other block
1600
- keyword = find_token(Kw, "do", consume: false)
1696
+ keyword = find_keyword(:do)
1601
1697
  if keyword &&
1602
1698
  keyword.location.start_char > collection.location.end_char &&
1603
1699
  keyword.location.end_char < ending.location.start_char
@@ -1645,8 +1741,8 @@ module SyntaxTree
1645
1741
  # :call-seq:
1646
1742
  # on_hash: ((nil | Array[AssocNew | AssocSplat]) assocs) -> HashLiteral
1647
1743
  def on_hash(assocs)
1648
- lbrace = find_token(LBrace)
1649
- rbrace = find_token(RBrace)
1744
+ lbrace = consume_token(LBrace)
1745
+ rbrace = consume_token(RBrace)
1650
1746
 
1651
1747
  HashLiteral.new(
1652
1748
  lbrace: lbrace,
@@ -1730,8 +1826,8 @@ module SyntaxTree
1730
1826
  if keyword_rest
1731
1827
  # We're doing this to delete the token from the list so that it doesn't
1732
1828
  # confuse future patterns by thinking they have an extra ** on the end.
1733
- find_token(Op, "**")
1734
- elsif (token = find_token(Op, "**", consume: false))
1829
+ consume_operator(:**)
1830
+ elsif (token = find_operator(:**))
1735
1831
  tokens.delete(token)
1736
1832
 
1737
1833
  # Create an artificial VarField if we find an extra ** on the end. This
@@ -1744,8 +1840,8 @@ module SyntaxTree
1744
1840
  # If there's no constant, there may be braces, so we're going to look for
1745
1841
  # those to get our bounds.
1746
1842
  unless constant
1747
- lbrace = find_token(LBrace, consume: false)
1748
- rbrace = find_token(RBrace, consume: false)
1843
+ lbrace = find_token(LBrace)
1844
+ rbrace = find_token(RBrace)
1749
1845
 
1750
1846
  if lbrace && rbrace
1751
1847
  parts = [lbrace, *parts, rbrace]
@@ -1784,8 +1880,8 @@ module SyntaxTree
1784
1880
  # (nil | Elsif | Else) consequent
1785
1881
  # ) -> If
1786
1882
  def on_if(predicate, statements, consequent)
1787
- beginning = find_token(Kw, "if")
1788
- ending = consequent || find_token(Kw, "end")
1883
+ beginning = consume_keyword(:if)
1884
+ ending = consequent || consume_keyword(:end)
1789
1885
 
1790
1886
  start_char = find_next_statement_start(predicate.location.end_char)
1791
1887
  statements.bind(
@@ -1817,7 +1913,7 @@ module SyntaxTree
1817
1913
  # :call-seq:
1818
1914
  # on_if_mod: (untyped predicate, untyped statement) -> IfMod
1819
1915
  def on_if_mod(predicate, statement)
1820
- find_token(Kw, "if")
1916
+ consume_keyword(:if)
1821
1917
 
1822
1918
  IfMod.new(
1823
1919
  statement: statement,
@@ -1860,11 +1956,11 @@ module SyntaxTree
1860
1956
  # Here we have a rightward assignment
1861
1957
  return pattern unless statements
1862
1958
 
1863
- beginning = find_token(Kw, "in")
1864
- ending = consequent || find_token(Kw, "end")
1959
+ beginning = consume_keyword(:in)
1960
+ ending = consequent || consume_keyword(:end)
1865
1961
 
1866
1962
  statements_start = pattern
1867
- if (token = find_token(Kw, "then", consume: false))
1963
+ if (token = find_keyword(:then))
1868
1964
  tokens.delete(token)
1869
1965
  statements_start = token
1870
1966
  end
@@ -1878,12 +1974,16 @@ module SyntaxTree
1878
1974
  ending.location.start_column
1879
1975
  )
1880
1976
 
1881
- In.new(
1882
- pattern: pattern,
1883
- statements: statements,
1884
- consequent: consequent,
1885
- location: beginning.location.to(ending.location)
1886
- )
1977
+ node =
1978
+ In.new(
1979
+ pattern: pattern,
1980
+ statements: statements,
1981
+ consequent: consequent,
1982
+ location: beginning.location.to(ending.location)
1983
+ )
1984
+
1985
+ PinVisitor.visit(node, tokens)
1986
+ node
1887
1987
  end
1888
1988
 
1889
1989
  # :call-seq:
@@ -1938,7 +2038,7 @@ module SyntaxTree
1938
2038
  # :call-seq:
1939
2039
  # on_kwrest_param: ((nil | Ident) name) -> KwRestParam
1940
2040
  def on_kwrest_param(name)
1941
- location = find_token(Op, "**").location
2041
+ location = consume_operator(:**).location
1942
2042
  location = location.to(name.location) if name
1943
2043
 
1944
2044
  KwRestParam.new(name: name, location: location)
@@ -1984,7 +2084,7 @@ module SyntaxTree
1984
2084
  # (BodyStmt | Statements) statements
1985
2085
  # ) -> Lambda
1986
2086
  def on_lambda(params, statements)
1987
- beginning = find_token(TLambda)
2087
+ beginning = consume_token(TLambda)
1988
2088
  braces =
1989
2089
  tokens.any? do |token|
1990
2090
  token.is_a?(TLamBeg) &&
@@ -1995,7 +2095,7 @@ module SyntaxTree
1995
2095
  # capturing lambda var until 3.2, we need to normalize all of that here.
1996
2096
  params =
1997
2097
  case params
1998
- in Paren[contents: Params]
2098
+ when Paren
1999
2099
  # In this case we've gotten to the <3.2 parentheses wrapping a set of
2000
2100
  # parameters case. Here we need to manually scan for lambda locals.
2001
2101
  range = (params.location.start_char + 1)...params.location.end_char
@@ -2015,23 +2115,23 @@ module SyntaxTree
2015
2115
  location: params.location,
2016
2116
  comments: params.comments
2017
2117
  )
2018
- in Params
2118
+ when Params
2019
2119
  # In this case we've gotten to the <3.2 plain set of parameters. In
2020
2120
  # this case there cannot be lambda locals, so we will wrap the
2021
2121
  # parameters into a lambda var that has no locals.
2022
2122
  LambdaVar.new(params: params, locals: [], location: params.location)
2023
- in LambdaVar
2123
+ when LambdaVar
2024
2124
  # In this case we've gotten to 3.2+ lambda var. In this case we don't
2025
2125
  # need to do anything and can just the value as given.
2026
2126
  params
2027
2127
  end
2028
2128
 
2029
2129
  if braces
2030
- opening = find_token(TLamBeg)
2031
- closing = find_token(RBrace)
2130
+ opening = consume_token(TLamBeg)
2131
+ closing = consume_token(RBrace)
2032
2132
  else
2033
- opening = find_token(Kw, "do")
2034
- closing = find_token(Kw, "end")
2133
+ opening = consume_keyword(:do)
2134
+ closing = consume_keyword(:end)
2035
2135
  end
2036
2136
 
2037
2137
  start_char = find_next_statement_start(opening.location.end_char)
@@ -2262,7 +2362,7 @@ module SyntaxTree
2262
2362
  # (nil | ARefField | Field | Ident | VarField) part
2263
2363
  # ) -> MLHS
2264
2364
  def on_mlhs_add_star(mlhs, part)
2265
- beginning = find_token(Op, "*")
2365
+ beginning = consume_operator(:*)
2266
2366
  ending = part || beginning
2267
2367
 
2268
2368
  location = beginning.location.to(ending.location)
@@ -2285,8 +2385,8 @@ module SyntaxTree
2285
2385
  # :call-seq:
2286
2386
  # on_mlhs_paren: ((MLHS | MLHSParen) contents) -> MLHSParen
2287
2387
  def on_mlhs_paren(contents)
2288
- lparen = find_token(LParen)
2289
- rparen = find_token(RParen)
2388
+ lparen = consume_token(LParen)
2389
+ rparen = consume_token(RParen)
2290
2390
 
2291
2391
  comma_range = lparen.location.end_char...rparen.location.start_char
2292
2392
  contents.comma = true if source[comma_range].strip.end_with?(",")
@@ -2303,8 +2403,8 @@ module SyntaxTree
2303
2403
  # BodyStmt bodystmt
2304
2404
  # ) -> ModuleDeclaration
2305
2405
  def on_module(constant, bodystmt)
2306
- beginning = find_token(Kw, "module")
2307
- ending = find_token(Kw, "end")
2406
+ beginning = consume_keyword(:module)
2407
+ ending = consume_keyword(:end)
2308
2408
  start_char = find_next_statement_start(constant.location.end_char)
2309
2409
 
2310
2410
  bodystmt.bind(
@@ -2343,7 +2443,7 @@ module SyntaxTree
2343
2443
  # :call-seq:
2344
2444
  # on_mrhs_add_star: (MRHS mrhs, untyped value) -> MRHS
2345
2445
  def on_mrhs_add_star(mrhs, value)
2346
- beginning = find_token(Op, "*")
2446
+ beginning = consume_operator(:*)
2347
2447
  ending = value || beginning
2348
2448
 
2349
2449
  arg_star =
@@ -2371,7 +2471,7 @@ module SyntaxTree
2371
2471
  # :call-seq:
2372
2472
  # on_next: (Args arguments) -> Next
2373
2473
  def on_next(arguments)
2374
- keyword = find_token(Kw, "next")
2474
+ keyword = consume_keyword(:next)
2375
2475
 
2376
2476
  location = keyword.location
2377
2477
  location = location.to(arguments.location) if arguments.parts.any?
@@ -2486,8 +2586,8 @@ module SyntaxTree
2486
2586
  # :call-seq:
2487
2587
  # on_paren: (untyped contents) -> Paren
2488
2588
  def on_paren(contents)
2489
- lparen = find_token(LParen)
2490
- rparen = find_token(RParen)
2589
+ lparen = consume_token(LParen)
2590
+ rparen = consume_token(RParen)
2491
2591
 
2492
2592
  if contents.is_a?(Params)
2493
2593
  location = contents.location
@@ -2551,13 +2651,13 @@ module SyntaxTree
2551
2651
  # :call-seq:
2552
2652
  # on_program: (Statements statements) -> Program
2553
2653
  def on_program(statements)
2554
- last_column = source.length - line_counts[lines.length - 1].start
2654
+ last_column = source.length - line_counts.last.start
2555
2655
  location =
2556
2656
  Location.new(
2557
2657
  start_line: 1,
2558
2658
  start_char: 0,
2559
2659
  start_column: 0,
2560
- end_line: lines.length,
2660
+ end_line: line_counts.length - 1,
2561
2661
  end_char: source.length,
2562
2662
  end_column: last_column
2563
2663
  )
@@ -2692,7 +2792,7 @@ module SyntaxTree
2692
2792
  # :call-seq:
2693
2793
  # on_qsymbols_new: () -> QSymbols
2694
2794
  def on_qsymbols_new
2695
- beginning = find_token(QSymbolsBeg)
2795
+ beginning = consume_token(QSymbolsBeg)
2696
2796
 
2697
2797
  QSymbols.new(
2698
2798
  beginning: beginning,
@@ -2733,7 +2833,7 @@ module SyntaxTree
2733
2833
  # :call-seq:
2734
2834
  # on_qwords_new: () -> QWords
2735
2835
  def on_qwords_new
2736
- beginning = find_token(QWordsBeg)
2836
+ beginning = consume_token(QWordsBeg)
2737
2837
 
2738
2838
  QWords.new(
2739
2839
  beginning: beginning,
@@ -2798,7 +2898,7 @@ module SyntaxTree
2798
2898
  # :call-seq:
2799
2899
  # on_redo: () -> Redo
2800
2900
  def on_redo
2801
- keyword = find_token(Kw, "redo")
2901
+ keyword = consume_keyword(:redo)
2802
2902
 
2803
2903
  Redo.new(value: keyword.value, location: keyword.location)
2804
2904
  end
@@ -2874,7 +2974,7 @@ module SyntaxTree
2874
2974
  # :call-seq:
2875
2975
  # on_regexp_new: () -> RegexpContent
2876
2976
  def on_regexp_new
2877
- regexp_beg = find_token(RegexpBeg)
2977
+ regexp_beg = consume_token(RegexpBeg)
2878
2978
 
2879
2979
  RegexpContent.new(
2880
2980
  beginning: regexp_beg.value,
@@ -2891,7 +2991,7 @@ module SyntaxTree
2891
2991
  # (nil | Rescue) consequent
2892
2992
  # ) -> Rescue
2893
2993
  def on_rescue(exceptions, variable, statements, consequent)
2894
- keyword = find_token(Kw, "rescue")
2994
+ keyword = consume_keyword(:rescue)
2895
2995
  exceptions = exceptions[0] if exceptions.is_a?(Array)
2896
2996
 
2897
2997
  last_node = variable || exceptions || keyword
@@ -2943,7 +3043,7 @@ module SyntaxTree
2943
3043
  # :call-seq:
2944
3044
  # on_rescue_mod: (untyped statement, untyped value) -> RescueMod
2945
3045
  def on_rescue_mod(statement, value)
2946
- find_token(Kw, "rescue")
3046
+ consume_keyword(:rescue)
2947
3047
 
2948
3048
  RescueMod.new(
2949
3049
  statement: statement,
@@ -2955,7 +3055,7 @@ module SyntaxTree
2955
3055
  # :call-seq:
2956
3056
  # on_rest_param: ((nil | Ident) name) -> RestParam
2957
3057
  def on_rest_param(name)
2958
- location = find_token(Op, "*").location
3058
+ location = consume_operator(:*).location
2959
3059
  location = location.to(name.location) if name
2960
3060
 
2961
3061
  RestParam.new(name: name, location: location)
@@ -2964,7 +3064,7 @@ module SyntaxTree
2964
3064
  # :call-seq:
2965
3065
  # on_retry: () -> Retry
2966
3066
  def on_retry
2967
- keyword = find_token(Kw, "retry")
3067
+ keyword = consume_keyword(:retry)
2968
3068
 
2969
3069
  Retry.new(value: keyword.value, location: keyword.location)
2970
3070
  end
@@ -2972,7 +3072,7 @@ module SyntaxTree
2972
3072
  # :call-seq:
2973
3073
  # on_return: (Args arguments) -> Return
2974
3074
  def on_return(arguments)
2975
- keyword = find_token(Kw, "return")
3075
+ keyword = consume_keyword(:return)
2976
3076
 
2977
3077
  Return.new(
2978
3078
  arguments: arguments,
@@ -2983,7 +3083,7 @@ module SyntaxTree
2983
3083
  # :call-seq:
2984
3084
  # on_return0: () -> Return0
2985
3085
  def on_return0
2986
- keyword = find_token(Kw, "return")
3086
+ keyword = consume_keyword(:return)
2987
3087
 
2988
3088
  Return0.new(value: keyword.value, location: keyword.location)
2989
3089
  end
@@ -3010,8 +3110,8 @@ module SyntaxTree
3010
3110
  # :call-seq:
3011
3111
  # on_sclass: (untyped target, BodyStmt bodystmt) -> SClass
3012
3112
  def on_sclass(target, bodystmt)
3013
- beginning = find_token(Kw, "class")
3014
- ending = find_token(Kw, "end")
3113
+ beginning = consume_keyword(:class)
3114
+ ending = consume_keyword(:end)
3015
3115
  start_char = find_next_statement_start(target.location.end_char)
3016
3116
 
3017
3117
  bodystmt.bind(
@@ -3109,7 +3209,7 @@ module SyntaxTree
3109
3209
  # :call-seq:
3110
3210
  # on_string_dvar: ((Backref | VarRef) variable) -> StringDVar
3111
3211
  def on_string_dvar(variable)
3112
- embvar = find_token(EmbVar)
3212
+ embvar = consume_token(EmbVar)
3113
3213
 
3114
3214
  StringDVar.new(
3115
3215
  variable: variable,
@@ -3120,8 +3220,8 @@ module SyntaxTree
3120
3220
  # :call-seq:
3121
3221
  # on_string_embexpr: (Statements statements) -> StringEmbExpr
3122
3222
  def on_string_embexpr(statements)
3123
- embexpr_beg = find_token(EmbExprBeg)
3124
- embexpr_end = find_token(EmbExprEnd)
3223
+ embexpr_beg = consume_token(EmbExprBeg)
3224
+ embexpr_end = consume_token(EmbExprEnd)
3125
3225
 
3126
3226
  statements.bind(
3127
3227
  embexpr_beg.location.end_char,
@@ -3162,8 +3262,8 @@ module SyntaxTree
3162
3262
  location: heredoc.location
3163
3263
  )
3164
3264
  else
3165
- tstring_beg = find_token(TStringBeg)
3166
- tstring_end = find_token(TStringEnd, location: tstring_beg.location)
3265
+ tstring_beg = consume_token(TStringBeg)
3266
+ tstring_end = consume_tstring_end(tstring_beg.location)
3167
3267
 
3168
3268
  location =
3169
3269
  Location.new(
@@ -3189,7 +3289,7 @@ module SyntaxTree
3189
3289
  # :call-seq:
3190
3290
  # on_super: ((ArgParen | Args) arguments) -> Super
3191
3291
  def on_super(arguments)
3192
- keyword = find_token(Kw, "super")
3292
+ keyword = consume_keyword(:super)
3193
3293
 
3194
3294
  Super.new(
3195
3295
  arguments: arguments,
@@ -3236,7 +3336,7 @@ module SyntaxTree
3236
3336
  # ) -> SymbolLiteral
3237
3337
  def on_symbol_literal(value)
3238
3338
  if value.is_a?(SymbolContent)
3239
- symbeg = find_token(SymBeg)
3339
+ symbeg = consume_token(SymBeg)
3240
3340
 
3241
3341
  SymbolLiteral.new(
3242
3342
  value: value.value,
@@ -3280,7 +3380,7 @@ module SyntaxTree
3280
3380
  # :call-seq:
3281
3381
  # on_symbols_new: () -> Symbols
3282
3382
  def on_symbols_new
3283
- beginning = find_token(SymbolsBeg)
3383
+ beginning = consume_token(SymbolsBeg)
3284
3384
 
3285
3385
  Symbols.new(
3286
3386
  beginning: beginning,
@@ -3410,13 +3510,13 @@ module SyntaxTree
3410
3510
  # We have somewhat special handling of the not operator since if it has
3411
3511
  # parentheses they don't get reported as a paren node for some reason.
3412
3512
 
3413
- beginning = find_token(Kw, "not")
3513
+ beginning = consume_keyword(:not)
3414
3514
  ending = statement || beginning
3415
3515
  parentheses = source[beginning.location.end_char] == "("
3416
3516
 
3417
3517
  if parentheses
3418
- find_token(LParen)
3419
- ending = find_token(RParen)
3518
+ consume_token(LParen)
3519
+ ending = consume_token(RParen)
3420
3520
  end
3421
3521
 
3422
3522
  Not.new(
@@ -3449,7 +3549,7 @@ module SyntaxTree
3449
3549
  # :call-seq:
3450
3550
  # on_undef: (Array[DynaSymbol | SymbolLiteral] symbols) -> Undef
3451
3551
  def on_undef(symbols)
3452
- keyword = find_token(Kw, "undef")
3552
+ keyword = consume_keyword(:undef)
3453
3553
 
3454
3554
  Undef.new(
3455
3555
  symbols: symbols,
@@ -3464,8 +3564,8 @@ module SyntaxTree
3464
3564
  # ((nil | Elsif | Else) consequent)
3465
3565
  # ) -> Unless
3466
3566
  def on_unless(predicate, statements, consequent)
3467
- beginning = find_token(Kw, "unless")
3468
- ending = consequent || find_token(Kw, "end")
3567
+ beginning = consume_keyword(:unless)
3568
+ ending = consequent || consume_keyword(:end)
3469
3569
 
3470
3570
  start_char = find_next_statement_start(predicate.location.end_char)
3471
3571
  statements.bind(
@@ -3486,7 +3586,7 @@ module SyntaxTree
3486
3586
  # :call-seq:
3487
3587
  # on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod
3488
3588
  def on_unless_mod(predicate, statement)
3489
- find_token(Kw, "unless")
3589
+ consume_keyword(:unless)
3490
3590
 
3491
3591
  UnlessMod.new(
3492
3592
  statement: statement,
@@ -3498,12 +3598,12 @@ module SyntaxTree
3498
3598
  # :call-seq:
3499
3599
  # on_until: (untyped predicate, Statements statements) -> Until
3500
3600
  def on_until(predicate, statements)
3501
- beginning = find_token(Kw, "until")
3502
- ending = find_token(Kw, "end")
3601
+ beginning = consume_keyword(:until)
3602
+ ending = consume_keyword(:end)
3503
3603
 
3504
3604
  # Consume the do keyword if it exists so that it doesn't get confused for
3505
3605
  # some other block
3506
- keyword = find_token(Kw, "do", consume: false)
3606
+ keyword = find_keyword(:do)
3507
3607
  if keyword && keyword.location.start_char > predicate.location.end_char &&
3508
3608
  keyword.location.end_char < ending.location.start_char
3509
3609
  tokens.delete(keyword)
@@ -3528,7 +3628,7 @@ module SyntaxTree
3528
3628
  # :call-seq:
3529
3629
  # on_until_mod: (untyped predicate, untyped statement) -> UntilMod
3530
3630
  def on_until_mod(predicate, statement)
3531
- find_token(Kw, "until")
3631
+ consume_keyword(:until)
3532
3632
 
3533
3633
  UntilMod.new(
3534
3634
  statement: statement,
@@ -3540,7 +3640,7 @@ module SyntaxTree
3540
3640
  # :call-seq:
3541
3641
  # on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias
3542
3642
  def on_var_alias(left, right)
3543
- keyword = find_token(Kw, "alias")
3643
+ keyword = consume_keyword(:alias)
3544
3644
 
3545
3645
  VarAlias.new(
3546
3646
  left: left,
@@ -3569,17 +3669,7 @@ module SyntaxTree
3569
3669
  # :call-seq:
3570
3670
  # on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef
3571
3671
  def on_var_ref(value)
3572
- pin = find_token(Op, "^", consume: false)
3573
-
3574
- if pin && pin.location.start_char == value.location.start_char - 1
3575
- tokens.delete(pin)
3576
- PinnedVarRef.new(
3577
- value: value,
3578
- location: pin.location.to(value.location)
3579
- )
3580
- else
3581
- VarRef.new(value: value, location: value.location)
3582
- end
3672
+ VarRef.new(value: value, location: value.location)
3583
3673
  end
3584
3674
 
3585
3675
  # :call-seq:
@@ -3604,11 +3694,11 @@ module SyntaxTree
3604
3694
  # (nil | Else | When) consequent
3605
3695
  # ) -> When
3606
3696
  def on_when(arguments, statements, consequent)
3607
- beginning = find_token(Kw, "when")
3608
- ending = consequent || find_token(Kw, "end")
3697
+ beginning = consume_keyword(:when)
3698
+ ending = consequent || consume_keyword(:end)
3609
3699
 
3610
3700
  statements_start = arguments
3611
- if (token = find_token(Kw, "then", consume: false))
3701
+ if (token = find_keyword(:then))
3612
3702
  tokens.delete(token)
3613
3703
  statements_start = token
3614
3704
  end
@@ -3634,12 +3724,12 @@ module SyntaxTree
3634
3724
  # :call-seq:
3635
3725
  # on_while: (untyped predicate, Statements statements) -> While
3636
3726
  def on_while(predicate, statements)
3637
- beginning = find_token(Kw, "while")
3638
- ending = find_token(Kw, "end")
3727
+ beginning = consume_keyword(:while)
3728
+ ending = consume_keyword(:end)
3639
3729
 
3640
3730
  # Consume the do keyword if it exists so that it doesn't get confused for
3641
3731
  # some other block
3642
- keyword = find_token(Kw, "do", consume: false)
3732
+ keyword = find_keyword(:do)
3643
3733
  if keyword && keyword.location.start_char > predicate.location.end_char &&
3644
3734
  keyword.location.end_char < ending.location.start_char
3645
3735
  tokens.delete(keyword)
@@ -3664,7 +3754,7 @@ module SyntaxTree
3664
3754
  # :call-seq:
3665
3755
  # on_while_mod: (untyped predicate, untyped statement) -> WhileMod
3666
3756
  def on_while_mod(predicate, statement)
3667
- find_token(Kw, "while")
3757
+ consume_keyword(:while)
3668
3758
 
3669
3759
  WhileMod.new(
3670
3760
  statement: statement,
@@ -3727,7 +3817,7 @@ module SyntaxTree
3727
3817
  # :call-seq:
3728
3818
  # on_words_new: () -> Words
3729
3819
  def on_words_new
3730
- beginning = find_token(WordsBeg)
3820
+ beginning = consume_token(WordsBeg)
3731
3821
 
3732
3822
  Words.new(
3733
3823
  beginning: beginning,
@@ -3761,7 +3851,7 @@ module SyntaxTree
3761
3851
  if heredoc && heredoc.beginning.value.include?("`")
3762
3852
  heredoc.location
3763
3853
  else
3764
- find_token(Backtick).location
3854
+ consume_token(Backtick).location
3765
3855
  end
3766
3856
 
3767
3857
  XString.new(parts: [], location: location)
@@ -3781,7 +3871,7 @@ module SyntaxTree
3781
3871
  location: heredoc.location
3782
3872
  )
3783
3873
  else
3784
- ending = find_token(TStringEnd, location: xstring.location)
3874
+ ending = consume_tstring_end(xstring.location)
3785
3875
 
3786
3876
  XStringLiteral.new(
3787
3877
  parts: xstring.parts,
@@ -3793,7 +3883,7 @@ module SyntaxTree
3793
3883
  # :call-seq:
3794
3884
  # on_yield: ((Args | Paren) arguments) -> Yield
3795
3885
  def on_yield(arguments)
3796
- keyword = find_token(Kw, "yield")
3886
+ keyword = consume_keyword(:yield)
3797
3887
 
3798
3888
  Yield.new(
3799
3889
  arguments: arguments,
@@ -3804,7 +3894,7 @@ module SyntaxTree
3804
3894
  # :call-seq:
3805
3895
  # on_yield0: () -> Yield0
3806
3896
  def on_yield0
3807
- keyword = find_token(Kw, "yield")
3897
+ keyword = consume_keyword(:yield)
3808
3898
 
3809
3899
  Yield0.new(value: keyword.value, location: keyword.location)
3810
3900
  end
@@ -3812,7 +3902,7 @@ module SyntaxTree
3812
3902
  # :call-seq:
3813
3903
  # on_zsuper: () -> ZSuper
3814
3904
  def on_zsuper
3815
- keyword = find_token(Kw, "super")
3905
+ keyword = consume_keyword(:super)
3816
3906
 
3817
3907
  ZSuper.new(value: keyword.value, location: keyword.location)
3818
3908
  end