ruby_parser 3.2.2 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (19) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.autotest +0 -2
  5. data/History.txt +38 -0
  6. data/Rakefile +2 -6
  7. data/lib/ruby18_parser.rb +2003 -2010
  8. data/lib/ruby18_parser.y +6 -8
  9. data/lib/ruby19_parser.rb +2016 -2004
  10. data/lib/ruby19_parser.y +6 -8
  11. data/lib/ruby20_parser.rb +2189 -2154
  12. data/lib/ruby20_parser.y +21 -13
  13. data/lib/ruby_lexer.rb +645 -812
  14. data/lib/ruby_parser_extras.rb +17 -46
  15. data/test/test_ruby_lexer.rb +1280 -1108
  16. data/test/test_ruby_parser.rb +101 -1
  17. data/test/test_ruby_parser_extras.rb +1 -63
  18. metadata +112 -133
  19. metadata.gz.sig +0 -0
data/lib/ruby_parser_extras.rb CHANGED
@@ -111,7 +111,7 @@ class RPStringScanner < StringScanner
111
111
  end
112
112
 
113
113
  module RubyParserStuff
114
- VERSION = "3.2.2" unless constants.include? "VERSION" # SIGH
114
+ VERSION = "3.3.0" unless constants.include? "VERSION" # SIGH
115
115
 
116
116
  attr_accessor :lexer, :in_def, :in_single, :file
117
117
  attr_reader :env, :comments
@@ -282,7 +282,7 @@ module RubyParserStuff
282
282
 
283
283
  def aryset receiver, index
284
284
  index ||= []
285
- s(:attrasgn, receiver, :"[]=", *index[1..-1])
285
+ s(:attrasgn, receiver, :"[]=", *index[1..-1]).compact # [][1..-1] => nil
286
286
  end
287
287
 
288
288
  def assignable(lhs, value = nil)
@@ -306,25 +306,14 @@ module RubyParserStuff
306
306
  s(:cdecl, id)
307
307
  else
308
308
  case self.env[id]
309
- when :lvar then
309
+ when :lvar, :dvar, nil then
310
310
  s(:lasgn, id)
311
- when :dvar, nil then
312
- if self.env.current[id] == :dvar then
313
- s(:lasgn, id)
314
- elsif self.env[id] == :dvar then
315
- self.env.use(id)
316
- s(:lasgn, id)
317
- elsif ! self.env.dynamic? then
318
- s(:lasgn, id)
319
- else
320
- s(:lasgn, id)
321
- end
322
311
  else
323
312
  raise "wtf? unknown type: #{self.env[id]}"
324
313
  end
325
314
  end
326
315
 
327
- self.env[id] ||= :lvar unless result.sexp_type == :cdecl # HACK? cdecl
316
+ self.env[id] ||= :lvar if result.sexp_type == :lasgn
328
317
 
329
318
  result << value if value
330
319
 
@@ -443,9 +432,11 @@ module RubyParserStuff
443
432
  def initialize(options = {})
444
433
  super()
445
434
 
446
- v = self.class.name[/1[89]/]
435
+ v = self.class.name[/1[89]|20/]
436
+
447
437
  self.lexer = RubyLexer.new v && v.to_i
448
438
  self.lexer.parser = self
439
+
449
440
  @env = RubyParserStuff::Environment.new
450
441
  @comments = []
451
442
 
@@ -562,7 +553,8 @@ module RubyParserStuff
562
553
 
563
554
  if elsebody and not resbody then
564
555
  warning("else without rescue is useless")
565
- result = block_append(s(:begin, result), elsebody)
556
+ result = s(:begin, result) if result
557
+ result = block_append(result, elsebody)
566
558
  end
567
559
 
568
560
  result = s(:ensure, result, ensurebody).compact if ensurebody
@@ -668,8 +660,10 @@ module RubyParserStuff
668
660
  end
669
661
  end
670
662
 
663
+ args.line line
671
664
  result.line = line
672
665
  result.comments = self.comments.pop
666
+
673
667
  result
674
668
  end
675
669
 
@@ -786,7 +780,9 @@ module RubyParserStuff
786
780
  }[c]
787
781
  raise "unknown regexp option: #{c}" unless v
788
782
  o += v
789
- k = c if c =~ /[esu]/
783
+
784
+ # encoding options are ignored on 1.9+
785
+ k = c if c =~ /[esu]/ if RUBY_VERSION < "1.9"
790
786
  end
791
787
 
792
788
  case node[0]
@@ -938,7 +934,9 @@ module RubyParserStuff
938
934
  case lhs[0]
939
935
  when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
940
936
  lhs << rhs
941
- when :attrasgn, :call then
937
+ when :attrasgn then
938
+ lhs << rhs
939
+ when :call then
942
940
  args = lhs.pop unless Symbol === lhs.last
943
941
  lhs.concat arg_add(args, rhs)[1..-1]
944
942
  when :const then
@@ -1236,55 +1234,28 @@ module RubyParserStuff
1236
1234
  @env.first
1237
1235
  end
1238
1236
 
1239
- def dynamic
1240
- idx = @dyn.index false
1241
- @env[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
1242
- end
1243
-
1244
- def dynamic?
1245
- @dyn[0] != false
1246
- end
1247
-
1248
1237
  def extend dyn = false
1249
1238
  @dyn.unshift dyn
1250
1239
  @env.unshift({})
1251
- @use.unshift({})
1252
1240
  end
1253
1241
 
1254
1242
  def initialize dyn = false
1255
1243
  @dyn = []
1256
1244
  @env = []
1257
- @use = []
1258
1245
  self.reset
1259
1246
  end
1260
1247
 
1261
1248
  def reset
1262
1249
  @dyn.clear
1263
1250
  @env.clear
1264
- @use.clear
1265
1251
  self.extend
1266
1252
  end
1267
1253
 
1268
1254
  def unextend
1269
1255
  @dyn.shift
1270
1256
  @env.shift
1271
- @use.shift
1272
1257
  raise "You went too far unextending env" if @env.empty?
1273
1258
  end
1274
-
1275
- def use id
1276
- @env.each_with_index do |env, i|
1277
- if env[id] then
1278
- @use[i][id] = true
1279
- end
1280
- end
1281
- end
1282
-
1283
- def used? id
1284
- idx = @dyn.index false # REFACTOR
1285
- u = @use[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
1286
- u[id]
1287
- end
1288
1259
  end
1289
1260
 
1290
1261
  class StackState
data/test/test_ruby_lexer.rb CHANGED
@@ -8,20 +8,20 @@ require 'ruby18_parser'
8
8
  require 'ruby20_parser'
9
9
 
10
10
  class TestRubyLexer < Minitest::Test
11
- attr_accessor :processor, :lex, :parser_class
11
+ attr_accessor :processor, :lex, :parser_class, :lex_state
12
12
 
13
13
  alias :lexer :lex # lets me copy/paste code from parser
14
14
  alias :lexer= :lex=
15
15
 
16
16
  def setup
17
+ self.lex_state = :expr_beg
17
18
  setup_lexer_class Ruby20Parser
18
19
  end
19
20
 
20
21
  def setup_lexer input, exp_sexp = nil
21
22
  setup_new_parser
22
23
  lex.src = input
23
- lex.lex_state = :expr_beg
24
- assert_equal exp_sexp, processor.class.new.parse(input) if exp_sexp
24
+ lex.lex_state = self.lex_state
25
25
  end
26
26
 
27
27
  def setup_new_parser
@@ -35,6 +35,117 @@ class TestRubyLexer < Minitest::Test
35
35
  setup_lexer "blah blah"
36
36
  end
37
37
 
38
+ def assert_lex input, exp_sexp, *args, &b
39
+ setup_lexer input
40
+ assert_parse input, exp_sexp if exp_sexp
41
+
42
+ b.call if b
43
+
44
+ args.each_slice(5) do |token, value, state, paren, brace|
45
+ assert_next_lexeme token, value, state, paren, brace
46
+ end
47
+
48
+ refute_lexeme
49
+ end
50
+
51
+ def assert_lex3 input, exp_sexp, *args, &block
52
+ args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
53
+
54
+ assert_lex(input, exp_sexp, *args, &block)
55
+ end
56
+
57
+ def refute_lex input, *args # TODO: re-sort
58
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
59
+
60
+ assert_raises RubyParser::SyntaxError do
61
+ assert_lex(input, nil, *args)
62
+ end
63
+ end
64
+
65
+ def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
66
+ assert_lex3("def #{name} ",
67
+ nil,
68
+
69
+ :kDEF, "def", :expr_fname,
70
+ type, name, end_state)
71
+ end
72
+
73
+ def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
74
+ adv = @lex.advance
75
+
76
+ assert adv, "no more tokens"
77
+
78
+ act_token, act_value = @lex.token, @lex.yacc_value
79
+
80
+ msg = message {
81
+ act = [act_token, act_value, @lex.lex_state,
82
+ @lex.paren_nest, @lex.brace_nest]
83
+ exp = [token, value, state, paren, brace]
84
+ "#{exp.inspect} vs #{act.inspect}"
85
+ }
86
+
87
+ act_value = act_value.first if Array === act_value
88
+
89
+ assert_equal token, act_token, msg
90
+ assert_equal value, act_value, msg
91
+ assert_equal state, @lex.lex_state, msg if state
92
+ assert_equal paren, @lex.paren_nest, msg if paren
93
+ assert_equal brace, @lex.brace_nest, msg if brace
94
+ end
95
+
96
+ def assert_parse input, exp_sexp
97
+ assert_equal exp_sexp, processor.class.new.parse(input)
98
+ end
99
+
100
+ def assert_read_escape expected, input
101
+ @lex.src = input
102
+ assert_equal expected, @lex.read_escape, input
103
+ end
104
+
105
+ def assert_read_escape_bad input # TODO: rename refute_read_escape
106
+ @lex.src = input
107
+ assert_raises RubyParser::SyntaxError do
108
+ @lex.read_escape
109
+ end
110
+ end
111
+
112
+ def refute_lexeme
113
+ x = @lex.advance
114
+ y = [@lex.token, @lex.yacc_value]
115
+ refute x, "not empty: #{y.inspect}"
116
+ end
117
+
118
+ ## Utility Methods:
119
+
120
+ def emulate_string_interpolation
121
+ lex_strterm = lexer.lex_strterm
122
+ string_nest = lexer.string_nest
123
+ brace_nest = lexer.brace_nest
124
+
125
+ lexer.string_nest = 0
126
+ lexer.brace_nest = 0
127
+ lexer.cond.push false
128
+ lexer.cmdarg.push false
129
+
130
+ lexer.lex_strterm = nil
131
+ lexer.lex_state = :expr_beg
132
+
133
+ yield
134
+
135
+ lexer.lex_state = :expr_endarg
136
+ assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
137
+
138
+ lexer.lex_strterm = lex_strterm
139
+ lexer.lex_state = :expr_beg
140
+ lexer.string_nest = string_nest
141
+ lexer.brace_nest = brace_nest
142
+
143
+ lexer.cond.lexpop
144
+ lexer.cmdarg.lexpop
145
+ end
146
+
147
+ ## Tests:
148
+
38
149
  def test_advance
39
150
  assert @lex.advance # blah
40
151
  assert @lex.advance # blah
@@ -43,214 +154,162 @@ class TestRubyLexer < Minitest::Test
43
154
 
44
155
  def test_unicode_ident
45
156
  s = "@\u1088\u1077\u1093\u1072"
46
- util_lex_token(s.dup,
47
- :tIVAR, s.dup)
157
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
48
158
  end
49
159
 
50
160
  def test_read_escape
51
- util_escape "\\", "\\"
52
- util_escape "\n", "n"
53
- util_escape "\t", "t"
54
- util_escape "\r", "r"
55
- util_escape "\f", "f"
56
- util_escape "\13", "v"
57
- util_escape "\0", "0"
58
- util_escape "\07", "a"
59
- util_escape "\007", "a"
60
- util_escape "\033", "e"
61
- util_escape "\377", "377"
62
- util_escape "\377", "xff"
63
- util_escape "\010", "b"
64
- util_escape " ", "s"
65
- util_escape "q", "q" # plain vanilla escape
66
-
67
- util_escape "8", "8" # ugh... mri... WHY?!?
68
- util_escape "9", "9" # ugh... mri... WHY?!?
69
-
70
- util_escape "$", "444" # ugh
161
+ assert_read_escape "\\", "\\"
162
+ assert_read_escape "\n", "n"
163
+ assert_read_escape "\t", "t"
164
+ assert_read_escape "\r", "r"
165
+ assert_read_escape "\f", "f"
166
+ assert_read_escape "\13", "v"
167
+ assert_read_escape "\0", "0"
168
+ assert_read_escape "\07", "a"
169
+ assert_read_escape "\007", "a"
170
+ assert_read_escape "\033", "e"
171
+ assert_read_escape "\377", "377"
172
+ assert_read_escape "\377", "xff"
173
+ assert_read_escape "\010", "b"
174
+ assert_read_escape " ", "s"
175
+ assert_read_escape "q", "q" # plain vanilla escape
176
+
177
+ assert_read_escape "8", "8" # ugh... mri... WHY?!?
178
+ assert_read_escape "9", "9" # ugh... mri... WHY?!?
179
+
180
+ assert_read_escape "$", "444" # ugh
71
181
  end
72
182
 
73
183
  def test_read_escape_c
74
- util_escape "\030", "C-x"
75
- util_escape "\030", "cx"
76
- util_escape "\230", 'C-\M-x'
77
- util_escape "\230", 'c\M-x'
184
+ assert_read_escape "\030", "C-x"
185
+ assert_read_escape "\030", "cx"
186
+ assert_read_escape "\230", 'C-\M-x'
187
+ assert_read_escape "\230", 'c\M-x'
78
188
 
79
- util_escape "\177", "C-?"
80
- util_escape "\177", "c?"
189
+ assert_read_escape "\177", "C-?"
190
+ assert_read_escape "\177", "c?"
81
191
  end
82
192
 
83
193
  def test_read_escape_errors
84
- util_escape_bad ""
194
+ assert_read_escape_bad ""
85
195
 
86
- util_escape_bad "M"
87
- util_escape_bad "M-"
88
- util_escape_bad "Mx"
196
+ assert_read_escape_bad "M"
197
+ assert_read_escape_bad "M-"
198
+ assert_read_escape_bad "Mx"
89
199
 
90
- util_escape_bad "Cx"
91
- util_escape_bad "C"
92
- util_escape_bad "C-"
200
+ assert_read_escape_bad "Cx"
201
+ assert_read_escape_bad "C"
202
+ assert_read_escape_bad "C-"
93
203
 
94
- util_escape_bad "c"
204
+ assert_read_escape_bad "c"
95
205
  end
96
206
 
97
207
  def test_read_escape_m
98
- util_escape "\370", "M-x"
99
- util_escape "\230", 'M-\C-x'
100
- util_escape "\230", 'M-\cx'
208
+ assert_read_escape "\370", "M-x"
209
+ assert_read_escape "\230", 'M-\C-x'
210
+ assert_read_escape "\230", 'M-\cx'
101
211
  end
102
212
 
103
213
  def test_yylex_ambiguous_uminus
104
- util_lex_token("m -3",
105
- :tIDENTIFIER, "m",
106
- :tUMINUS_NUM, "-",
107
- :tINTEGER, 3)
214
+ assert_lex3("m -3",
215
+ nil,
216
+ :tIDENTIFIER, "m", :expr_cmdarg,
217
+ :tUMINUS_NUM, "-", :expr_beg,
218
+ :tINTEGER, 3, :expr_end)
219
+
108
220
  # TODO: verify warning
109
221
  end
110
222
 
111
223
  def test_yylex_ambiguous_uplus
112
- util_lex_token("m +3",
113
- :tIDENTIFIER, "m",
114
- :tINTEGER, 3)
224
+ assert_lex3("m +3",
225
+ nil,
226
+ :tIDENTIFIER, "m", :expr_cmdarg,
227
+ :tINTEGER, 3, :expr_end)
228
+
115
229
  # TODO: verify warning
116
230
  end
117
231
 
118
232
  def test_yylex_and
119
- util_lex_token "&", :tAMPER, "&"
233
+ assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
120
234
  end
121
235
 
122
236
  def test_yylex_and2
123
- util_lex_token "&&", :tANDOP, "&&"
237
+ assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
124
238
  end
125
239
 
126
240
  def test_yylex_and2_equals
127
- util_lex_token "&&=", :tOP_ASGN, "&&"
241
+ assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
128
242
  end
129
243
 
130
244
  def test_yylex_and_arg
131
- @lex.lex_state = :expr_arg
245
+ self.lex_state = :expr_arg
132
246
 
133
- util_lex_token(" &y",
134
- :tAMPER, "&",
135
- :tIDENTIFIER, "y")
247
+ assert_lex3(" &y",
248
+ nil,
249
+ :tAMPER, "&", :expr_beg,
250
+ :tIDENTIFIER, "y", :expr_arg)
136
251
  end
137
252
 
138
253
  def test_yylex_and_equals
139
- util_lex_token "&=", :tOP_ASGN, "&"
254
+ assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
140
255
  end
141
256
 
142
257
  def test_yylex_and_expr
143
- @lex.lex_state = :expr_arg
258
+ self.lex_state = :expr_arg
144
259
 
145
- util_lex_token("x & y",
146
- :tIDENTIFIER, "x",
147
- :tAMPER2, "&",
148
- :tIDENTIFIER, "y")
260
+ assert_lex3("x & y",
261
+ nil,
262
+ :tIDENTIFIER, "x", :expr_cmdarg,
263
+ :tAMPER2, "&", :expr_beg,
264
+ :tIDENTIFIER, "y", :expr_arg)
149
265
  end
150
266
 
151
267
  def test_yylex_and_meth
152
- util_lex_fname "&", :tAMPER2
268
+ assert_lex_fname "&", :tAMPER2
153
269
  end
154
270
 
155
271
  def test_yylex_assoc
156
- util_lex_token "=>", :tASSOC, "=>"
272
+ assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
157
273
  end
158
274
 
159
275
  def test_yylex_label__18
160
276
  setup_lexer_class Ruby18Parser
161
277
 
162
- util_lex_token("{a:",
163
- :tLBRACE, "{",
164
- :tIDENTIFIER, "a",
165
- :tSYMBEG, ":")
278
+ assert_lex3("{a:",
279
+ nil,
280
+ :tLBRACE, "{", :expr_beg,
281
+ :tIDENTIFIER, "a", :expr_arg,
282
+ :tSYMBEG, ":", :expr_fname)
166
283
  end
167
284
 
168
285
  def test_yylex_label_in_params__18
169
286
  setup_lexer_class Ruby18Parser
170
287
 
171
- util_lex_token("foo(a:",
172
- :tIDENTIFIER, "foo",
173
- :tLPAREN2, "(",
174
- :tIDENTIFIER, "a",
175
- :tSYMBEG, ":")
288
+ assert_lex3("foo(a:",
289
+ nil,
290
+ :tIDENTIFIER, "foo", :expr_cmdarg,
291
+ :tLPAREN2, "(", :expr_beg,
292
+ :tIDENTIFIER, "a", :expr_cmdarg,
293
+ :tSYMBEG, ":", :expr_fname)
176
294
  end
177
295
 
178
296
  def test_yylex_label__19
179
297
  setup_lexer_class Ruby19Parser
180
298
 
181
- util_lex_token("{a:",
182
- :tLBRACE, "{",
183
- :tLABEL, "a")
299
+ assert_lex3("{a:",
300
+ nil,
301
+ :tLBRACE, "{", :expr_beg,
302
+ :tLABEL, "a", :expr_beg)
184
303
  end
185
304
 
186
305
  def test_yylex_label_in_params__19
187
306
  setup_lexer_class Ruby19Parser
188
307
 
189
- util_lex_token("foo(a:",
190
- :tIDENTIFIER, "foo",
191
- :tLPAREN2, "(",
192
- :tLABEL, "a")
193
- end
194
-
195
- def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
196
- assert @lex.advance, "no more tokens"
197
-
198
- msg = message {
199
- act = [@lex.token, @lex.yacc_value, @lex.lex_state,
200
- @lex.paren_nest, @lex.brace_nest]
201
- exp = [token, value, state, paren, brace]
202
- "#{exp.inspect} vs #{act.inspect}"
203
- }
204
-
205
- act_value = @lex.yacc_value
206
- act_value = act_value.first if Array === act_value
207
-
208
- assert_equal token, @lex.token, msg
209
- assert_equal value, act_value, msg
210
- assert_equal state, @lex.lex_state, msg
211
- assert_equal paren, @lex.paren_nest, msg if paren
212
- assert_equal brace, @lex.brace_nest, msg if brace
213
- end
214
-
215
- def refute_lexeme
216
- refute @lex.advance, "not empty: #{[@lex.token, @lex.yacc_value].inspect}"
217
- end
218
-
219
- def assert_lex input, exp_sexp, *args
220
- setup_lexer input, exp_sexp
221
-
222
- args.each_slice(5) do |token, value, state, paren, brace|
223
- assert_next_lexeme token, value, state, paren, brace
224
- end
225
-
226
- refute_lexeme
227
- end
228
-
229
- def emulate_string_interpolation
230
- lex_strterm = lexer.lex_strterm
231
- string_nest = lexer.string_nest
232
- brace_nest = lexer.brace_nest
233
-
234
- lexer.string_nest = 0
235
- lexer.brace_nest = 0
236
- lexer.cond.push false
237
- lexer.cmdarg.push false
238
-
239
- lexer.lex_strterm = nil
240
- lexer.lex_state = :expr_beg
241
-
242
- yield
243
-
244
- lexer.lex_state = :expr_endarg
245
- assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
246
-
247
- lexer.lex_strterm = lex_strterm
248
- lexer.lex_state = :expr_beg
249
- lexer.string_nest = string_nest
250
- lexer.brace_nest = brace_nest
251
-
252
- lexer.cond.lexpop
253
- lexer.cmdarg.lexpop
308
+ assert_lex3("foo(a:",
309
+ nil,
310
+ :tIDENTIFIER, "foo", :expr_cmdarg,
311
+ :tLPAREN2, "(", :expr_beg,
312
+ :tLABEL, "a", :expr_beg)
254
313
  end
255
314
 
256
315
  def test_yylex_paren_string_parens_interpolated
@@ -532,1839 +591,2000 @@ class TestRubyLexer < Minitest::Test
532
591
  end
533
592
 
534
593
  def test_yylex_back_ref
535
- util_lex_token("[$&, $`, $', $+]",
536
- :tLBRACK, "[",
537
- :tBACK_REF, :"&", :tCOMMA, ",",
538
- :tBACK_REF, :"`", :tCOMMA, ",",
539
- :tBACK_REF, :"'", :tCOMMA, ",",
540
- :tBACK_REF, :"+",
541
- :tRBRACK, "]")
594
+ assert_lex3("[$&, $`, $', $+]",
595
+ nil,
596
+ :tLBRACK, "[", :expr_beg,
597
+ :tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
598
+ :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
599
+ :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
600
+ :tBACK_REF, :+, :expr_end,
601
+ :tRBRACK, "]", :expr_endarg)
542
602
  end
543
603
 
544
604
  def test_yylex_backslash
545
- util_lex_token("1 \\\n+ 2",
546
- :tINTEGER, 1,
547
- :tPLUS, "+",
548
- :tINTEGER, 2)
605
+ assert_lex3("1 \\\n+ 2",
606
+ nil,
607
+ :tINTEGER, 1, :expr_end,
608
+ :tPLUS, "+", :expr_beg,
609
+ :tINTEGER, 2, :expr_end)
549
610
  end
550
611
 
551
612
  def test_yylex_backslash_bad
552
- util_bad_token("1 \\ + 2",
553
- :tINTEGER, 1)
613
+ refute_lex("1 \\ + 2", :tINTEGER, 1)
554
614
  end
555
615
 
556
616
  def test_yylex_backtick
557
- util_lex_token("`ls`",
558
- :tXSTRING_BEG, "`",
559
- :tSTRING_CONTENT, "ls",
560
- :tSTRING_END, "`")
617
+ assert_lex3("`ls`",
618
+ nil,
619
+ :tXSTRING_BEG, "`", :expr_beg,
620
+ :tSTRING_CONTENT, "ls", :expr_beg,
621
+ :tSTRING_END, "`", :expr_end)
561
622
  end
562
623
 
563
624
  def test_yylex_backtick_cmdarg
564
- @lex.lex_state = :expr_dot
565
- util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
625
+ self.lex_state = :expr_dot
566
626
 
567
- assert_equal :expr_cmdarg, @lex.lex_state
627
+ # \n ensures expr_cmd (TODO: why?)
628
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
568
629
  end
569
630
 
570
631
  def test_yylex_backtick_dot
571
- @lex.lex_state = :expr_dot
572
- util_lex_token("a.`(3)",
573
- :tIDENTIFIER, "a",
574
- :tDOT, ".",
575
- :tBACK_REF2, "`",
576
- :tLPAREN2, "(",
577
- :tINTEGER, 3,
578
- :tRPAREN, ")")
632
+ self.lex_state = :expr_dot
633
+
634
+ assert_lex3("a.`(3)",
635
+ nil,
636
+ :tIDENTIFIER, "a", :expr_cmdarg,
637
+ :tDOT, ".", :expr_dot,
638
+ :tBACK_REF2, "`", :expr_arg,
639
+ :tLPAREN2, "(", :expr_beg,
640
+ :tINTEGER, 3, :expr_end,
641
+ :tRPAREN, ")", :expr_endfn)
579
642
  end
580
643
 
581
644
  def test_yylex_backtick_method
582
- @lex.lex_state = :expr_fname
583
- util_lex_token("`", :tBACK_REF2, "`")
584
- assert_equal :expr_end, @lex.lex_state
645
+ self.lex_state = :expr_fname
646
+
647
+ assert_lex3("`",
648
+ nil,
649
+ :tBACK_REF2, "`", :expr_end)
585
650
  end
586
651
 
587
652
  def test_yylex_bad_char
588
- util_bad_token(" \010 ")
653
+ refute_lex(" \010 ")
589
654
  end
590
655
 
591
656
  def test_yylex_bang
592
- util_lex_token "!", :tBANG, "!"
657
+ assert_lex3("!", nil, :tBANG, "!", :expr_beg)
593
658
  end
594
659
 
595
660
  def test_yylex_bang_equals
596
- util_lex_token "!=", :tNEQ, "!="
661
+ assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
597
662
  end
598
663
 
599
664
  def test_yylex_bang_tilde
600
- util_lex_token "!~", :tNMATCH, "!~"
665
+ assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
601
666
  end
602
667
 
603
668
  def test_yylex_carat
604
- util_lex_token "^", :tCARET, "^"
669
+ assert_lex3("^", nil, :tCARET, "^", :expr_beg)
605
670
  end
606
671
 
607
672
  def test_yylex_carat_equals
608
- util_lex_token "^=", :tOP_ASGN, "^"
673
+ assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
609
674
  end
610
675
 
611
676
  def test_yylex_colon2
612
- util_lex_token("A::B",
613
- :tCONSTANT, "A",
614
- :tCOLON2, "::",
615
- :tCONSTANT, "B")
677
+ assert_lex3("A::B",
678
+ nil,
679
+ :tCONSTANT, "A", :expr_cmdarg,
680
+ :tCOLON2, "::", :expr_dot,
681
+ :tCONSTANT, "B", :expr_arg)
682
+ end
683
+
684
+ def test_yylex_colon2_argh
685
+ assert_lex3("module X::Y\n c\nend",
686
+ nil,
687
+ :kMODULE, "module", :expr_value,
688
+ :tCONSTANT, "X", :expr_arg,
689
+ :tCOLON2, "::", :expr_dot,
690
+ :tCONSTANT, "Y", :expr_arg,
691
+ :tNL, nil, :expr_beg,
692
+ :tIDENTIFIER, "c", :expr_cmdarg,
693
+ :tNL, nil, :expr_beg,
694
+ :kEND, "end", :expr_end)
616
695
  end
617
696
 
618
697
  def test_yylex_colon3
619
- util_lex_token("::Array",
620
- :tCOLON3, "::",
621
- :tCONSTANT, "Array")
698
+ assert_lex3("::Array",
699
+ nil,
700
+ :tCOLON3, "::", :expr_beg,
701
+ :tCONSTANT, "Array", :expr_arg)
622
702
  end
623
703
 
624
704
  def test_yylex_comma
625
- util_lex_token ",", :tCOMMA, ","
705
+ assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
626
706
  end
627
707
 
628
708
  def test_yylex_comment
629
- util_lex_token("1 # one\n# two\n2",
630
- :tINTEGER, 1,
631
- :tNL, nil,
632
- :tINTEGER, 2)
709
+ assert_lex3("1 # one\n# two\n2",
710
+ nil,
711
+ :tINTEGER, 1, :expr_end,
712
+ :tNL, nil, :expr_beg,
713
+ :tINTEGER, 2, :expr_end)
714
+
633
715
  assert_equal "# one\n# two\n", @lex.comments
634
716
  end
635
717
 
636
718
  def test_yylex_comment_begin
637
- util_lex_token("=begin\nblah\nblah\n=end\n42",
638
- :tINTEGER, 42)
719
+ assert_lex3("=begin\nblah\nblah\n=end\n42",
720
+ nil,
721
+ :tINTEGER, 42, :expr_end)
722
+
639
723
  assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
640
724
  end
641
725
 
642
726
  def test_yylex_comment_begin_bad
643
- util_bad_token("=begin\nblah\nblah\n")
727
+ refute_lex("=begin\nblah\nblah\n")
728
+
644
729
  assert_equal "", @lex.comments
645
730
  end
646
731
 
647
732
  def test_yylex_comment_begin_not_comment
648
- util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
649
- :tIDENTIFIER, "beginfoo",
650
- :tEQL, "=",
651
- :tINTEGER, 5,
652
- :tNL, nil,
653
- :tIDENTIFIER, "p",
654
- :tIDENTIFIER, "x",
655
- :tEQL, "=",
656
- :tIDENTIFIER, "beginfoo")
733
+ assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
734
+ nil,
735
+ :tIDENTIFIER, "beginfoo", :expr_cmdarg,
736
+ :tEQL, "=", :expr_beg,
737
+ :tINTEGER, 5, :expr_end,
738
+ :tNL, nil, :expr_beg,
739
+ :tIDENTIFIER, "p", :expr_cmdarg,
740
+ :tIDENTIFIER, "x", :expr_arg,
741
+ :tEQL, "=", :expr_beg,
742
+ :tIDENTIFIER, "beginfoo", :expr_arg)
657
743
  end
658
744
 
659
745
  def test_yylex_comment_begin_space
660
- util_lex_token("=begin blah\nblah\n=end\n")
746
+ assert_lex3("=begin blah\nblah\n=end\n", nil)
747
+
661
748
  assert_equal "=begin blah\nblah\n=end\n", @lex.comments
662
749
  end
663
750
 
664
751
  def test_yylex_comment_end_space_and_text
665
- util_lex_token("=begin blah\nblah\n=end blab\n")
752
+ assert_lex3("=begin blah\nblah\n=end blab\n", nil)
753
+
666
754
  assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
667
755
  end
668
756
 
669
757
  def test_yylex_comment_eos
670
- util_lex_token("# comment")
758
+ assert_lex3("# comment", nil)
671
759
  end
672
760
 
673
761
  def test_yylex_constant
674
- util_lex_token("ArgumentError",
675
- :tCONSTANT, "ArgumentError")
762
+ assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
676
763
  end
677
764
 
678
765
  def test_yylex_constant_semi
679
- util_lex_token("ArgumentError;",
680
- :tCONSTANT, "ArgumentError",
681
- :tSEMI, ";")
766
+ assert_lex3("ArgumentError;",
767
+ nil,
768
+ :tCONSTANT, "ArgumentError", :expr_cmdarg,
769
+ :tSEMI, ";", :expr_beg)
682
770
  end
683
771
 
684
772
  def test_yylex_cvar
685
- util_lex_token "@@blah", :tCVAR, "@@blah"
773
+ assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
686
774
  end
687
775
 
688
776
  def test_yylex_cvar_bad
689
777
  assert_raises RubyParser::SyntaxError do
690
- util_lex_token "@@1"
778
+ assert_lex3("@@1", nil)
691
779
  end
692
780
  end
693
781
 
694
782
  def test_yylex_def_bad_name
695
- @lex.lex_state = :expr_fname
696
- util_bad_token("def [ ", :kDEF, "def")
783
+ self.lex_state = :expr_fname
784
+ refute_lex("def [ ", :kDEF, "def")
697
785
  end
698
786
 
699
787
  def test_yylex_div
700
- util_lex_token("a / 2",
701
- :tIDENTIFIER, "a",
702
- :tDIVIDE, "/",
703
- :tINTEGER, 2)
788
+ assert_lex3("a / 2",
789
+ nil,
790
+ :tIDENTIFIER, "a", :expr_cmdarg,
791
+ :tDIVIDE, "/", :expr_beg,
792
+ :tINTEGER, 2, :expr_end)
704
793
  end
705
794
 
706
795
  def test_yylex_div_equals
707
- util_lex_token("a /= 2",
708
- :tIDENTIFIER, "a",
709
- :tOP_ASGN, "/",
710
- :tINTEGER, 2)
796
+ assert_lex3("a /= 2",
797
+ nil,
798
+ :tIDENTIFIER, "a", :expr_cmdarg,
799
+ :tOP_ASGN, "/", :expr_beg,
800
+ :tINTEGER, 2, :expr_end)
711
801
  end
712
802
 
713
803
  def test_yylex_do
714
- util_lex_token("x do 42 end",
715
- :tIDENTIFIER, "x",
716
- :kDO, "do",
717
- :tINTEGER, 42,
718
- :kEND, "end")
804
+ assert_lex3("x do 42 end",
805
+ nil,
806
+ :tIDENTIFIER, "x", :expr_cmdarg,
807
+ :kDO, "do", :expr_beg,
808
+ :tINTEGER, 42, :expr_end,
809
+ :kEND, "end", :expr_end)
719
810
  end
720
811
 
721
812
  def test_yylex_do_block
722
- @lex.lex_state = :expr_endarg
723
- @lex.cmdarg.push true
724
-
725
- util_lex_token("x.y do 42 end",
726
- :tIDENTIFIER, "x",
727
- :tDOT, ".",
728
- :tIDENTIFIER, "y",
729
- :kDO_BLOCK, "do",
730
- :tINTEGER, 42,
731
- :kEND, "end")
813
+ self.lex_state = :expr_endarg
814
+
815
+ assert_lex3("x.y do 42 end",
816
+ nil,
817
+ :tIDENTIFIER, "x", :expr_end,
818
+ :tDOT, ".", :expr_dot,
819
+ :tIDENTIFIER, "y", :expr_arg,
820
+ :kDO_BLOCK, "do", :expr_beg,
821
+ :tINTEGER, 42, :expr_end,
822
+ :kEND, "end", :expr_end) do
823
+ @lex.cmdarg.push true
824
+ end
732
825
  end
733
826
 
734
827
  def test_yylex_do_block2
735
- @lex.lex_state = :expr_endarg
828
+ self.lex_state = :expr_endarg
736
829
 
737
- util_lex_token("do 42 end",
738
- :kDO_BLOCK, "do",
739
- :tINTEGER, 42,
740
- :kEND, "end")
830
+ assert_lex3("do 42 end",
831
+ nil,
832
+ :kDO_BLOCK, "do", :expr_beg,
833
+ :tINTEGER, 42, :expr_end,
834
+ :kEND, "end", :expr_end)
741
835
  end
742
836
 
743
837
  def test_yylex_do_cond
744
- @lex.cond.push true
745
-
746
- util_lex_token("x do 42 end",
747
- :tIDENTIFIER, "x",
748
- :kDO_COND, "do",
749
- :tINTEGER, 42,
750
- :kEND, "end")
838
+ assert_lex3("x do 42 end",
839
+ nil,
840
+ :tIDENTIFIER, "x", :expr_cmdarg,
841
+ :kDO_COND, "do", :expr_beg,
842
+ :tINTEGER, 42, :expr_end,
843
+ :kEND, "end", :expr_end) do
844
+ @lex.cond.push true
845
+ end
751
846
  end
752
847
 
753
848
  def test_yylex_dollar
754
- util_lex_token("$", "$", "$") # FIX: wtf is this?!?
849
+ assert_lex3("$", nil, "$", "$", :expr_end)
850
+ # FIX: wtf is this?!?
755
851
  end
756
852
 
757
853
  def test_yylex_dot # HINT message sends
758
- util_lex_token ".", :tDOT, "."
854
+ assert_lex3(".", nil, :tDOT, ".", :expr_dot)
759
855
  end
760
856
 
761
857
  def test_yylex_dot2
762
- util_lex_token "..", :tDOT2, ".."
858
+ assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
763
859
  end
764
860
 
765
861
  def test_yylex_dot3
766
- util_lex_token "...", :tDOT3, "..."
862
+ assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
767
863
  end
768
864
 
769
865
  def test_yylex_equals
770
- util_lex_token "=", :tEQL, "=" # FIX: this sucks
866
+ # FIX: this sucks
867
+ assert_lex3("=", nil, :tEQL, "=", :expr_beg)
771
868
  end
772
869
 
773
870
  def test_yylex_equals2
774
- util_lex_token "==", :tEQ, "=="
871
+ assert_lex3("==", nil, :tEQ, "==", :expr_beg)
775
872
  end
776
873
 
777
874
  def test_yylex_equals3
778
- util_lex_token "===", :tEQQ, "==="
875
+ assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
779
876
  end
780
877
 
781
878
  def test_yylex_equals_tilde
782
- util_lex_token "=~", :tMATCH, "=~"
879
+ assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
783
880
  end
784
881
 
785
882
  def test_yylex_float
786
- util_lex_token "1.0", :tFLOAT, 1.0
883
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
787
884
  end
788
885
 
789
886
  def test_yylex_float_bad_no_underscores
790
- util_bad_token "1__0.0"
887
+ refute_lex "1__0.0"
791
888
  end
792
889
 
793
890
  def test_yylex_float_bad_no_zero_leading
794
- util_bad_token ".0"
891
+ refute_lex ".0"
795
892
  end
796
893
 
797
894
  def test_yylex_float_bad_trailing_underscore
798
- util_bad_token "123_.0"
895
+ refute_lex "123_.0"
799
896
  end
800
897
 
801
898
  def test_yylex_float_call
802
- util_lex_token("1.0.to_s",
803
- :tFLOAT, 1.0,
804
- :tDOT, ".",
805
- :tIDENTIFIER, "to_s")
899
+ assert_lex3("1.0.to_s",
900
+ nil,
901
+ :tFLOAT, 1.0, :expr_end,
902
+ :tDOT, ".", :expr_dot,
903
+ :tIDENTIFIER, "to_s", :expr_arg)
806
904
  end
807
905
 
808
906
  def test_yylex_float_dot_E
809
- util_lex_token "1.0E10", :tFLOAT, 1.0e10
907
+ assert_lex3("1.0E10",
908
+ nil,
909
+ :tFLOAT, 10000000000.0, :expr_end)
810
910
  end
811
911
 
812
912
  def test_yylex_float_dot_E_neg
813
- util_lex_token("-1.0E10",
814
- :tUMINUS_NUM, "-",
815
- :tFLOAT, 1.0e10)
913
+ assert_lex3("-1.0E10",
914
+ nil,
915
+ :tUMINUS_NUM, "-", :expr_beg,
916
+ :tFLOAT, 10000000000.0, :expr_end)
816
917
  end
817
918
 
818
919
  def test_yylex_float_dot_e
819
- util_lex_token "1.0e10", :tFLOAT, 1.0e10
920
+ assert_lex3("1.0e10",
921
+ nil,
922
+ :tFLOAT, 10000000000.0, :expr_end)
820
923
  end
821
924
 
822
925
  def test_yylex_float_dot_e_neg
823
- util_lex_token("-1.0e10",
824
- :tUMINUS_NUM, "-",
825
- :tFLOAT, 1.0e10)
926
+ assert_lex3("-1.0e10",
927
+ nil,
928
+ :tUMINUS_NUM, "-", :expr_beg,
929
+ :tFLOAT, 10000000000.0, :expr_end)
826
930
  end
827
931
 
828
932
  def test_yylex_float_e
829
- util_lex_token "1e10", :tFLOAT, 1e10
933
+ assert_lex3("1e10",
934
+ nil,
935
+ :tFLOAT, 10000000000.0, :expr_end)
830
936
  end
831
937
 
832
938
  def test_yylex_float_e_bad_double_e
833
- util_bad_token "1e2e3"
939
+ refute_lex "1e2e3"
834
940
  end
835
941
 
836
942
  def test_yylex_float_e_bad_trailing_underscore
837
- util_bad_token "123_e10"
943
+ refute_lex "123_e10"
838
944
  end
839
945
 
840
946
  def test_yylex_float_e_minus
841
- util_lex_token "1e-10", :tFLOAT, 1e-10
947
+ assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
842
948
  end
843
949
 
844
950
  def test_yylex_float_e_neg
845
- util_lex_token("-1e10",
846
- :tUMINUS_NUM, "-",
847
- :tFLOAT, 1e10)
951
+ assert_lex3("-1e10",
952
+ nil,
953
+ :tUMINUS_NUM, "-", :expr_beg,
954
+ :tFLOAT, 10000000000.0, :expr_end)
848
955
  end
849
956
 
850
957
  def test_yylex_float_e_neg_minus
851
- util_lex_token("-1e-10",
852
- :tUMINUS_NUM, "-",
853
- :tFLOAT, 1e-10)
958
+ assert_lex3("-1e-10",
959
+ nil,
960
+ :tUMINUS_NUM, "-", :expr_beg,
961
+ :tFLOAT, 1.0e-10, :expr_end)
854
962
  end
855
963
 
856
964
  def test_yylex_float_e_neg_plus
857
- util_lex_token("-1e+10",
858
- :tUMINUS_NUM, "-",
859
- :tFLOAT, 1e10)
965
+ assert_lex3("-1e+10",
966
+ nil,
967
+ :tUMINUS_NUM, "-", :expr_beg,
968
+ :tFLOAT, 10000000000.0, :expr_end)
860
969
  end
861
970
 
862
971
  def test_yylex_float_e_plus
863
- util_lex_token "1e+10", :tFLOAT, 1e10
972
+ assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
864
973
  end
865
974
 
866
975
  def test_yylex_float_e_zero
867
- util_lex_token "0e0", :tFLOAT, 0e0
976
+ assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
868
977
  end
869
978
 
870
979
  def test_yylex_float_neg
871
- util_lex_token("-1.0",
872
- :tUMINUS_NUM, "-",
873
- :tFLOAT, 1.0)
980
+ assert_lex3("-1.0",
981
+ nil,
982
+ :tUMINUS_NUM, "-", :expr_beg,
983
+ :tFLOAT, 1.0, :expr_end)
874
984
  end
875
985
 
876
986
  def test_yylex_ge
877
- util_lex_token("a >= 2",
878
- :tIDENTIFIER, "a",
879
- :tGEQ, ">=",
880
- :tINTEGER, 2)
987
+ assert_lex3("a >= 2",
988
+ nil,
989
+ :tIDENTIFIER, "a", :expr_cmdarg,
990
+ :tGEQ, ">=", :expr_beg,
991
+ :tINTEGER, 2, :expr_end)
881
992
  end
882
993
 
883
994
  def test_yylex_global
884
- util_lex_token("$blah", :tGVAR, "$blah")
995
+ assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
885
996
  end
886
997
 
887
998
  def test_yylex_global_backref
888
- @lex.lex_state = :expr_fname
889
- util_lex_token("$`", :tGVAR, "$`")
999
+ self.lex_state = :expr_fname
1000
+
1001
+ assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
890
1002
  end
891
1003
 
892
1004
  def test_yylex_global_dash_nothing
893
- util_lex_token("$- ", :tGVAR, "$-")
1005
+ assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
894
1006
  end
895
1007
 
896
1008
  def test_yylex_global_dash_something
897
- util_lex_token("$-x", :tGVAR, "$-x")
1009
+ assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
898
1010
  end
899
1011
 
900
1012
  def test_yylex_global_number
901
- @lex.lex_state = :expr_fname
902
- util_lex_token("$1", :tGVAR, "$1")
1013
+ self.lex_state = :expr_fname
1014
+
1015
+ assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
903
1016
  end
904
1017
 
905
1018
  def test_yylex_global_number_big
906
- @lex.lex_state = :expr_fname
907
- util_lex_token("$1234", :tGVAR, "$1234")
1019
+ self.lex_state = :expr_fname
1020
+
1021
+ assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
908
1022
  end
909
1023
 
910
1024
  def test_yylex_global_other
911
- util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
912
- :tLBRACK, "[",
913
- :tGVAR, "$~", :tCOMMA, ",",
914
- :tGVAR, "$*", :tCOMMA, ",",
915
- :tGVAR, "$$", :tCOMMA, ",",
916
- :tGVAR, "$\?", :tCOMMA, ",",
917
- :tGVAR, "$!", :tCOMMA, ",",
918
- :tGVAR, "$@", :tCOMMA, ",",
919
- :tGVAR, "$/", :tCOMMA, ",",
920
- :tGVAR, "$\\", :tCOMMA, ",",
921
- :tGVAR, "$;", :tCOMMA, ",",
922
- :tGVAR, "$,", :tCOMMA, ",",
923
- :tGVAR, "$.", :tCOMMA, ",",
924
- :tGVAR, "$=", :tCOMMA, ",",
925
- :tGVAR, "$:", :tCOMMA, ",",
926
- :tGVAR, "$<", :tCOMMA, ",",
927
- :tGVAR, "$>", :tCOMMA, ",",
928
- :tGVAR, "$\"",
929
- :tRBRACK, "]")
1025
+ assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
1026
+ nil,
1027
+ :tLBRACK, "[", :expr_beg,
1028
+ :tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
1029
+ :tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
1030
+ :tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
1031
+ :tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
1032
+ :tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
1033
+ :tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
1034
+ :tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
1035
+ :tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
1036
+ :tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
1037
+ :tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
1038
+ :tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
1039
+ :tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
1040
+ :tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
1041
+ :tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
1042
+ :tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
1043
+ :tGVAR, "$\"", :expr_end,
1044
+ :tRBRACK, "]", :expr_endarg)
930
1045
  end
931
1046
 
932
1047
  def test_yylex_global_underscore
933
- util_lex_token("$_",
934
- :tGVAR, "$_")
1048
+ assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
935
1049
  end
936
1050
 
937
1051
  def test_yylex_global_wierd
938
- util_lex_token("$__blah",
939
- :tGVAR, "$__blah")
1052
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
940
1053
  end
941
1054
 
942
1055
  def test_yylex_global_zero
943
- util_lex_token("$0", :tGVAR, "$0")
1056
+ assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
944
1057
  end
945
1058
 
946
1059
  def test_yylex_gt
947
- util_lex_token("a > 2",
948
- :tIDENTIFIER, "a",
949
- :tGT, ">",
950
- :tINTEGER, 2)
1060
+ assert_lex3("a > 2",
1061
+ nil,
1062
+ :tIDENTIFIER, "a", :expr_cmdarg,
1063
+ :tGT, ">", :expr_beg,
1064
+ :tINTEGER, 2, :expr_end)
951
1065
  end
952
1066
 
953
1067
  def test_yylex_heredoc_backtick
954
- util_lex_token("a = <<`EOF`\n blah blah\nEOF\n",
955
- :tIDENTIFIER, "a",
956
- :tEQL, "=",
957
- :tXSTRING_BEG, "`",
958
- :tSTRING_CONTENT, " blah blah\n",
959
- :tSTRING_END, "EOF",
960
- :tNL, nil)
1068
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1069
+ nil,
1070
+ :tIDENTIFIER, "a", :expr_cmdarg,
1071
+ :tEQL, "=", :expr_beg,
1072
+ :tXSTRING_BEG, "`", :expr_beg,
1073
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1074
+ :tSTRING_END, "EOF", :expr_end,
1075
+ :tNL, nil, :expr_beg)
961
1076
  end
962
1077
 
963
1078
  def test_yylex_heredoc_double
964
- util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n",
965
- :tIDENTIFIER, "a",
966
- :tEQL, "=",
967
- :tSTRING_BEG, "\"",
968
- :tSTRING_CONTENT, " blah blah\n",
969
- :tSTRING_END, "EOF",
970
- :tNL, nil)
1079
+ assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1080
+ nil,
1081
+ :tIDENTIFIER, "a", :expr_cmdarg,
1082
+ :tEQL, "=", :expr_beg,
1083
+ :tSTRING_BEG, "\"", :expr_beg,
1084
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1085
+ :tSTRING_END, "EOF", :expr_end,
1086
+ :tNL, nil, :expr_beg)
971
1087
  end
972
1088
 
973
1089
  def test_yylex_heredoc_double_dash
974
- util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n",
975
- :tIDENTIFIER, "a",
976
- :tEQL, "=",
977
- :tSTRING_BEG, "\"",
978
- :tSTRING_CONTENT, " blah blah\n",
979
- :tSTRING_END, "EOF",
980
- :tNL, nil)
1090
+ assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1091
+ nil,
1092
+ :tIDENTIFIER, "a", :expr_cmdarg,
1093
+ :tEQL, "=", :expr_beg,
1094
+ :tSTRING_BEG, "\"", :expr_beg,
1095
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1096
+ :tSTRING_END, "EOF", :expr_end,
1097
+ :tNL, nil, :expr_beg)
981
1098
  end
982
1099
 
983
1100
  def test_yylex_heredoc_double_eos
984
- util_bad_token("a = <<\"EOF\"\nblah",
985
- :tIDENTIFIER, "a",
986
- :tEQL, "=",
987
- :tSTRING_BEG, "\"")
1101
+ refute_lex("a = <<\"EOF\"\nblah",
1102
+ :tIDENTIFIER, "a",
1103
+ :tEQL, "=",
1104
+ :tSTRING_BEG, "\"")
988
1105
  end
989
1106
 
990
1107
  def test_yylex_heredoc_double_eos_nl
991
- util_bad_token("a = <<\"EOF\"\nblah\n",
992
- :tIDENTIFIER, "a",
993
- :tEQL, "=",
994
- :tSTRING_BEG, "\"")
1108
+ refute_lex("a = <<\"EOF\"\nblah\n",
1109
+ :tIDENTIFIER, "a",
1110
+ :tEQL, "=",
1111
+ :tSTRING_BEG, "\"")
995
1112
  end
996
1113
 
997
1114
  def test_yylex_heredoc_double_interp
998
- # TODO: convert to assert_lex
999
- util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
1000
- :tIDENTIFIER, "a",
1001
- :tEQL, "=",
1002
- :tSTRING_BEG, "\"",
1003
- :tSTRING_CONTENT, "#x a ",
1004
- :tSTRING_DVAR, "\#@",
1005
- :tSTRING_CONTENT, "@a b ", # HUH?
1006
- :tSTRING_DVAR, "\#$",
1007
- :tSTRING_CONTENT, "$b c ", # HUH?
1008
- :tSTRING_DBEG, "\#{",
1009
- :tSTRING_CONTENT, "3} \n", # HUH?
1010
- :tSTRING_END, "EOF",
1011
- :tNL, nil)
1115
+ assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
1116
+ nil,
1117
+ :tIDENTIFIER, "a", :expr_cmdarg,
1118
+ :tEQL, "=", :expr_beg,
1119
+ :tSTRING_BEG, "\"", :expr_beg,
1120
+ :tSTRING_CONTENT, "#x a ", :expr_beg,
1121
+ :tSTRING_DVAR, "\#@", :expr_beg,
1122
+ :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
1123
+ :tSTRING_DVAR, "\#$", :expr_beg,
1124
+ :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
1125
+ :tSTRING_DBEG, "\#{", :expr_beg,
1126
+ :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
1127
+ :tSTRING_END, "EOF", :expr_end,
1128
+ :tNL, nil, :expr_beg)
1012
1129
  end
1013
1130
 
1014
1131
  def test_yylex_heredoc_empty
1015
- util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
1016
- :tSTRING_BEG, "\"",
1017
- :tSTRING_DBEG, "\#{",
1018
- :tSTRING_CONTENT, "x}\nblah2\n",
1019
- :tSTRING_END, "",
1020
- :tNL, nil)
1132
+ assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1133
+ nil,
1134
+ :tSTRING_BEG, "\"", :expr_beg,
1135
+ :tSTRING_DBEG, "\#{", :expr_beg,
1136
+ :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
1137
+ :tSTRING_END, "", :expr_end,
1138
+ :tNL, nil, :expr_beg)
1021
1139
  end
1022
1140
 
1023
1141
  def test_yylex_heredoc_none
1024
- util_lex_token("a = <<EOF\nblah\nblah\nEOF",
1025
- :tIDENTIFIER, "a",
1026
- :tEQL, "=",
1027
- :tSTRING_BEG, "\"",
1028
- :tSTRING_CONTENT, "blah\nblah\n",
1029
- :tSTRING_END, "EOF",
1030
- :tNL, nil)
1142
+ assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
1143
+ nil,
1144
+ :tIDENTIFIER, "a", :expr_cmdarg,
1145
+ :tEQL, "=", :expr_beg,
1146
+ :tSTRING_BEG, "\"", :expr_beg,
1147
+ :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1148
+ :tSTRING_END, "EOF", :expr_end,
1149
+ :tNL, nil, :expr_beg)
1031
1150
  end
1032
1151
 
1033
1152
  def test_yylex_heredoc_none_bad_eos
1034
- util_bad_token("a = <<EOF",
1035
- :tIDENTIFIER, "a",
1036
- :tEQL, "=",
1037
- :tSTRING_BEG, "\"")
1153
+ refute_lex("a = <<EOF",
1154
+ :tIDENTIFIER, "a",
1155
+ :tEQL, "=",
1156
+ :tSTRING_BEG, "\"")
1038
1157
  end
1039
1158
 
1040
1159
  def test_yylex_heredoc_none_dash
1041
- util_lex_token("a = <<-EOF\nblah\nblah\n EOF",
1042
- :tIDENTIFIER, "a",
1043
- :tEQL, "=",
1044
- :tSTRING_BEG, "\"",
1045
- :tSTRING_CONTENT, "blah\nblah\n",
1046
- :tSTRING_END, "EOF",
1047
- :tNL, nil)
1160
+ assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
1161
+ nil,
1162
+ :tIDENTIFIER, "a", :expr_cmdarg,
1163
+ :tEQL, "=", :expr_beg,
1164
+ :tSTRING_BEG, "\"", :expr_beg,
1165
+ :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1166
+ :tSTRING_END, "EOF", :expr_end,
1167
+ :tNL, nil, :expr_beg)
1048
1168
  end
1049
1169
 
1050
1170
  def test_yylex_heredoc_single
1051
- util_lex_token("a = <<'EOF'\n blah blah\nEOF\n",
1052
- :tIDENTIFIER, "a",
1053
- :tEQL, "=",
1054
- :tSTRING_BEG, "\"",
1055
- :tSTRING_CONTENT, " blah blah\n",
1056
- :tSTRING_END, "EOF",
1057
- :tNL, nil)
1171
+ assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1172
+ nil,
1173
+ :tIDENTIFIER, "a", :expr_cmdarg,
1174
+ :tEQL, "=", :expr_beg,
1175
+ :tSTRING_BEG, "\"", :expr_beg,
1176
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1177
+ :tSTRING_END, "EOF", :expr_end,
1178
+ :tNL, nil, :expr_beg)
1058
1179
  end
1059
1180
 
1060
1181
  def test_yylex_heredoc_single_bad_eos_body
1061
- util_bad_token("a = <<'EOF'\nblah",
1062
- :tIDENTIFIER, "a",
1063
- :tEQL, "=",
1064
- :tSTRING_BEG, "\"")
1182
+ refute_lex("a = <<'EOF'\nblah",
1183
+ :tIDENTIFIER, "a",
1184
+ :tEQL, "=",
1185
+ :tSTRING_BEG, "\"")
1065
1186
  end
1066
1187
 
1067
1188
  def test_yylex_heredoc_single_bad_eos_empty
1068
- util_bad_token("a = <<''\n",
1069
- :tIDENTIFIER, "a",
1070
- :tEQL, "=",
1071
- :tSTRING_BEG, "\"")
1189
+ refute_lex("a = <<''\n",
1190
+ :tIDENTIFIER, "a",
1191
+ :tEQL, "=",
1192
+ :tSTRING_BEG, "\"")
1072
1193
  end
1073
1194
 
1074
1195
  def test_yylex_heredoc_single_bad_eos_term
1075
- util_bad_token("a = <<'EOF",
1076
- :tIDENTIFIER, "a",
1077
- :tEQL, "=",
1078
- :tSTRING_BEG, "\"")
1196
+ refute_lex("a = <<'EOF",
1197
+ :tIDENTIFIER, "a",
1198
+ :tEQL, "=",
1199
+ :tSTRING_BEG, "\"")
1079
1200
  end
1080
1201
 
1081
1202
  def test_yylex_heredoc_single_bad_eos_term_nl
1082
- util_bad_token("a = <<'EOF\ns = 'blah blah'",
1083
- :tIDENTIFIER, "a",
1084
- :tEQL, "=",
1085
- :tSTRING_BEG, "\"")
1203
+ refute_lex("a = <<'EOF\ns = 'blah blah'",
1204
+ :tIDENTIFIER, "a",
1205
+ :tEQL, "=",
1206
+ :tSTRING_BEG, "\"")
1086
1207
  end
1087
1208
 
1088
1209
  def test_yylex_heredoc_single_dash
1089
- util_lex_token("a = <<-'EOF'\n blah blah\n EOF\n",
1090
- :tIDENTIFIER, "a",
1091
- :tEQL, "=",
1092
- :tSTRING_BEG, "\"",
1093
- :tSTRING_CONTENT, " blah blah\n",
1094
- :tSTRING_END, "EOF",
1095
- :tNL, nil)
1210
+ assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
1211
+ nil,
1212
+ :tIDENTIFIER, "a", :expr_cmdarg,
1213
+ :tEQL, "=", :expr_beg,
1214
+ :tSTRING_BEG, "\"", :expr_beg,
1215
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1216
+ :tSTRING_END, "EOF", :expr_end,
1217
+ :tNL, nil, :expr_beg)
1096
1218
  end
1097
1219
 
1098
1220
  def test_yylex_identifier
1099
- util_lex_token("identifier", :tIDENTIFIER, "identifier")
1221
+ assert_lex3("identifier",
1222
+ nil,
1223
+ :tIDENTIFIER, "identifier", :expr_cmdarg)
1100
1224
  end
1101
1225
 
1102
1226
  def test_yylex_identifier_bang
1103
- util_lex_token("identifier!", :tFID, "identifier!")
1227
+ assert_lex3("identifier!",
1228
+ nil,
1229
+ :tFID, "identifier!", :expr_cmdarg)
1104
1230
  end
1105
1231
 
1106
1232
  def test_yylex_identifier_cmp
1107
- util_lex_fname "<=>", :tCMP
1233
+ assert_lex_fname "<=>", :tCMP
1108
1234
  end
1109
1235
 
1110
1236
  def test_yylex_identifier_def__18
1111
1237
  setup_lexer_class Ruby18Parser
1112
1238
 
1113
- util_lex_fname "identifier", :tIDENTIFIER, :expr_end
1239
+ assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
1114
1240
  end
1115
1241
 
1116
1242
  def test_yylex_identifier_def__1920
1117
1243
  setup_lexer_class Ruby19Parser
1118
1244
 
1119
- util_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1245
+ assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1120
1246
  end
1121
1247
 
1122
1248
  def test_yylex_identifier_eh
1123
- util_lex_token("identifier?", :tFID, "identifier?")
1249
+ assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
1124
1250
  end
1125
1251
 
1126
1252
  def test_yylex_identifier_equals_arrow
1127
- @lex.lex_state = :expr_fname
1128
- util_lex_token(":blah==>",
1129
- :tSYMBOL, "blah=",
1130
- :tASSOC, "=>")
1253
+ assert_lex3(":blah==>",
1254
+ nil,
1255
+ :tSYMBOL, "blah=", :expr_end,
1256
+ :tASSOC, "=>", :expr_beg)
1131
1257
  end
1132
1258
 
1133
1259
  def test_yylex_identifier_equals3
1134
- # @lex.lex_state = :expr_fname
1135
- util_lex_token(":a===b",
1136
- :tSYMBOL, "a",
1137
- :tEQQ, "===",
1138
- :tIDENTIFIER, "b")
1260
+ assert_lex3(":a===b",
1261
+ nil,
1262
+ :tSYMBOL, "a", :expr_end,
1263
+ :tEQQ, "===", :expr_beg,
1264
+ :tIDENTIFIER, "b", :expr_arg)
1139
1265
  end
1140
1266
 
1141
1267
  def test_yylex_identifier_equals_equals_arrow
1142
- # @lex.lex_state = :expr_fname
1143
- util_lex_token(":a==>b",
1144
- :tSYMBOL, "a=",
1145
- :tASSOC, "=>",
1146
- :tIDENTIFIER, "b")
1268
+ assert_lex3(":a==>b",
1269
+ nil,
1270
+ :tSYMBOL, "a=", :expr_end,
1271
+ :tASSOC, "=>", :expr_beg,
1272
+ :tIDENTIFIER, "b", :expr_arg)
1147
1273
  end
1148
1274
 
1149
1275
  def test_yylex_identifier_equals_caret
1150
- util_lex_fname "^", :tCARET
1276
+ assert_lex_fname "^", :tCARET
1151
1277
  end
1152
1278
 
1153
1279
  def test_yylex_identifier_equals_def__18
1154
1280
  setup_lexer_class Ruby18Parser
1155
1281
 
1156
- util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
1282
+ assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
1157
1283
  end
1158
1284
 
1159
1285
  def test_yylex_identifier_equals_def__1920
1160
1286
  setup_lexer_class Ruby19Parser
1161
1287
 
1162
- util_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1288
+ assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1163
1289
  end
1164
1290
 
1165
1291
  def test_yylex_identifier_equals_def2
1166
- util_lex_fname "==", :tEQ
1292
+ assert_lex_fname "==", :tEQ
1167
1293
  end
1168
1294
 
1169
1295
  def test_yylex_identifier_equals_expr
1170
- @lex.lex_state = :expr_dot
1171
- util_lex_token("y = arg",
1172
- :tIDENTIFIER, "y",
1173
- :tEQL, "=",
1174
- :tIDENTIFIER, "arg")
1175
-
1176
- assert_equal :expr_arg, @lex.lex_state
1296
+ self.lex_state = :expr_dot
1297
+ assert_lex3("y = arg",
1298
+ nil,
1299
+ :tIDENTIFIER, "y", :expr_cmdarg,
1300
+ :tEQL, "=", :expr_beg,
1301
+ :tIDENTIFIER, "arg", :expr_arg)
1177
1302
  end
1178
1303
 
1179
1304
  def test_yylex_identifier_equals_or
1180
- util_lex_fname "|", :tPIPE
1305
+ assert_lex_fname "|", :tPIPE
1181
1306
  end
1182
1307
 
1183
1308
  def test_yylex_identifier_equals_slash
1184
- util_lex_fname "/", :tDIVIDE
1309
+ assert_lex_fname "/", :tDIVIDE
1185
1310
  end
1186
1311
 
1187
1312
  def test_yylex_identifier_equals_tilde
1188
- @lex.lex_state = :expr_fname # can only set via parser's defs
1189
- util_lex_token("identifier=~",
1190
- :tIDENTIFIER, "identifier",
1191
- :tMATCH, "=~")
1313
+ self.lex_state = :expr_fname # can only set via parser's defs
1314
+
1315
+ assert_lex3("identifier=~",
1316
+ nil,
1317
+ :tIDENTIFIER, "identifier", :expr_endfn,
1318
+ :tMATCH, "=~", :expr_beg)
1192
1319
  end
1193
1320
 
1194
1321
  def test_yylex_identifier_gt
1195
- util_lex_fname ">", :tGT
1322
+ assert_lex_fname ">", :tGT
1196
1323
  end
1197
1324
 
1198
1325
  def test_yylex_identifier_le
1199
- util_lex_fname "<=", :tLEQ
1326
+ assert_lex_fname "<=", :tLEQ
1200
1327
  end
1201
1328
 
1202
1329
  def test_yylex_identifier_lt
1203
- util_lex_fname "<", :tLT
1330
+ assert_lex_fname "<", :tLT
1204
1331
  end
1205
1332
 
1206
1333
  def test_yylex_identifier_tilde
1207
- util_lex_fname "~", :tTILDE
1334
+ assert_lex_fname "~", :tTILDE
1208
1335
  end
1209
1336
 
1210
1337
  def test_yylex_index
1211
- util_lex_fname "[]", :tAREF
1338
+ assert_lex_fname "[]", :tAREF
1212
1339
  end
1213
1340
 
1214
1341
  def test_yylex_index_equals
1215
- util_lex_fname "[]=", :tASET
1342
+ assert_lex_fname "[]=", :tASET
1216
1343
  end
1217
1344
 
1218
1345
  def test_yylex_integer
1219
- util_lex_token "42", :tINTEGER, 42
1346
+ assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1220
1347
  end
1221
1348
 
1222
1349
  def test_yylex_integer_bin
1223
- util_lex_token "0b101010", :tINTEGER, 42
1350
+ assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
1224
1351
  end
1225
1352
 
1226
1353
  def test_yylex_integer_bin_bad_none
1227
- util_bad_token "0b "
1354
+ refute_lex "0b "
1228
1355
  end
1229
1356
 
1230
1357
  def test_yylex_integer_bin_bad_underscores
1231
- util_bad_token "0b10__01"
1358
+ refute_lex "0b10__01"
1232
1359
  end
1233
1360
 
1234
1361
  def test_yylex_integer_dec
1235
- util_lex_token "42", :tINTEGER, 42
1362
+ assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1236
1363
  end
1237
1364
 
1238
1365
  def test_yylex_integer_dec_bad_underscores
1239
- util_bad_token "42__24"
1366
+ refute_lex "42__24"
1240
1367
  end
1241
1368
 
1242
1369
  def test_yylex_integer_dec_d
1243
- util_lex_token "0d42", :tINTEGER, 42
1370
+ assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
1244
1371
  end
1245
1372
 
1246
1373
  def test_yylex_integer_dec_d_bad_none
1247
- util_bad_token "0d"
1374
+ refute_lex "0d"
1248
1375
  end
1249
1376
 
1250
1377
  def test_yylex_integer_dec_d_bad_underscores
1251
- util_bad_token "0d42__24"
1378
+ refute_lex "0d42__24"
1252
1379
  end
1253
1380
 
1254
1381
  def test_yylex_question_eh_a__18
1255
1382
  setup_lexer_class Ruby18Parser
1256
1383
 
1257
- util_lex_token "?a", :tINTEGER, 97
1384
+ assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
1258
1385
  end
1259
1386
 
1260
1387
  def test_yylex_question_eh_a__19
1261
1388
  setup_lexer_class Ruby19Parser
1262
1389
 
1263
- util_lex_token '?a', :tSTRING, "a"
1390
+ assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
1264
1391
  end
1265
1392
 
1266
1393
  def test_yylex_question_eh_escape_M_escape_C__18
1267
1394
  setup_lexer_class Ruby18Parser
1268
1395
 
1269
- util_lex_token '?\M-\C-a', :tINTEGER, 129
1396
+ assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
1270
1397
  end
1271
1398
 
1272
1399
  def test_yylex_question_eh_escape_M_escape_C__19
1273
1400
  setup_lexer_class Ruby19Parser
1274
1401
 
1275
- util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
1402
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
1276
1403
  end
1277
1404
 
1278
1405
  def test_yylex_integer_hex
1279
- util_lex_token "0x2a", :tINTEGER, 42
1406
+ assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
1280
1407
  end
1281
1408
 
1282
1409
  def test_yylex_integer_hex_bad_none
1283
- util_bad_token "0x "
1410
+ refute_lex "0x "
1284
1411
  end
1285
1412
 
1286
1413
  def test_yylex_integer_hex_bad_underscores
1287
- util_bad_token "0xab__cd"
1414
+ refute_lex "0xab__cd"
1288
1415
  end
1289
1416
 
1290
1417
  def test_yylex_integer_oct
1291
- util_lex_token "052", :tINTEGER, 42
1418
+ assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
1292
1419
  end
1293
1420
 
1294
1421
  def test_yylex_integer_oct_bad_range
1295
- util_bad_token "08"
1422
+ refute_lex "08"
1296
1423
  end
1297
1424
 
1298
1425
  def test_yylex_integer_oct_bad_range2
1299
- util_bad_token "08"
1426
+ refute_lex "08"
1300
1427
  end
1301
1428
 
1302
1429
  def test_yylex_integer_oct_bad_underscores
1303
- util_bad_token "01__23"
1430
+ refute_lex "01__23"
1304
1431
  end
1305
1432
 
1306
1433
  def test_yylex_integer_oct_O
1307
- util_lex_token "0O52", :tINTEGER, 42
1434
+ assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
1308
1435
  end
1309
1436
 
1310
1437
  def test_yylex_integer_oct_O_bad_range
1311
- util_bad_token "0O8"
1438
+ refute_lex "0O8"
1312
1439
  end
1313
1440
 
1314
1441
  def test_yylex_integer_oct_O_bad_underscores
1315
- util_bad_token "0O1__23"
1442
+ refute_lex "0O1__23"
1316
1443
  end
1317
1444
 
1318
1445
  def test_yylex_integer_oct_O_not_bad_none
1319
- util_lex_token "0O ", :tINTEGER, 0
1446
+ assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
1320
1447
  end
1321
1448
 
1322
1449
  def test_yylex_integer_oct_o
1323
- util_lex_token "0o52", :tINTEGER, 42
1450
+ assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
1324
1451
  end
1325
1452
 
1326
1453
  def test_yylex_integer_oct_o_bad_range
1327
- util_bad_token "0o8"
1454
+ refute_lex "0o8"
1328
1455
  end
1329
1456
 
1330
1457
  def test_yylex_integer_oct_o_bad_underscores
1331
- util_bad_token "0o1__23"
1458
+ refute_lex "0o1__23"
1332
1459
  end
1333
1460
 
1334
1461
  def test_yylex_integer_oct_o_not_bad_none
1335
- util_lex_token "0o ", :tINTEGER, 0
1462
+ assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
1336
1463
  end
1337
1464
 
1338
1465
  def test_yylex_integer_trailing
1339
- util_lex_token("1.to_s",
1340
- :tINTEGER, 1,
1341
- :tDOT, '.',
1342
- :tIDENTIFIER, 'to_s')
1466
+ assert_lex3("1.to_s",
1467
+ nil,
1468
+ :tINTEGER, 1, :expr_end,
1469
+ :tDOT, ".", :expr_dot,
1470
+ :tIDENTIFIER, "to_s", :expr_arg)
1343
1471
  end
1344
1472
 
1345
1473
  def test_yylex_integer_underscore
1346
- util_lex_token "4_2", :tINTEGER, 42
1474
+ assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
1347
1475
  end
1348
1476
 
1349
1477
  def test_yylex_integer_underscore_bad
1350
- util_bad_token "4__2"
1478
+ refute_lex "4__2"
1351
1479
  end
1352
1480
 
1353
1481
  def test_yylex_integer_zero
1354
- util_lex_token "0", :tINTEGER, 0
1482
+ assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
1355
1483
  end
1356
1484
 
1357
1485
  def test_yylex_ivar
1358
- util_lex_token "@blah", :tIVAR, "@blah"
1486
+ assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
1359
1487
  end
1360
1488
 
1361
1489
  def test_yylex_ivar_bad
1362
- util_bad_token "@1"
1490
+ refute_lex "@1"
1363
1491
  end
1364
1492
 
1365
1493
  def test_yylex_ivar_bad_0_length
1366
- util_bad_token "1+@\n", :tINTEGER, 1, :tPLUS, "+"
1494
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
1367
1495
  end
1368
1496
 
1369
1497
  def test_yylex_keyword_expr
1370
- @lex.lex_state = :expr_endarg
1371
-
1372
- util_lex_token("if", :kIF_MOD, "if")
1498
+ self.lex_state = :expr_endarg
1373
1499
 
1374
- assert_equal :expr_beg, @lex.lex_state
1500
+ assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
1375
1501
  end
1376
1502
 
1377
1503
  def test_yylex_lt
1378
- util_lex_token "<", :tLT, "<"
1504
+ assert_lex3("<", nil, :tLT, "<", :expr_beg)
1379
1505
  end
1380
1506
 
1381
1507
  def test_yylex_lt2
1382
- util_lex_token("a <\< b",
1383
- :tIDENTIFIER, "a",
1384
- :tLSHFT, "<\<",
1385
- :tIDENTIFIER, "b")
1386
-
1508
+ assert_lex3("a << b",
1509
+ nil,
1510
+ :tIDENTIFIER, "a", :expr_cmdarg,
1511
+ :tLSHFT, "<<", :expr_beg,
1512
+ :tIDENTIFIER, "b", :expr_arg)
1387
1513
  end
1388
1514
 
1389
1515
  def test_yylex_lt2_equals
1390
- util_lex_token("a <\<= b",
1391
- :tIDENTIFIER, "a",
1392
- :tOP_ASGN, "<\<",
1393
- :tIDENTIFIER, "b")
1516
+ assert_lex3("a <<= b",
1517
+ nil,
1518
+ :tIDENTIFIER, "a", :expr_cmdarg,
1519
+ :tOP_ASGN, "<<", :expr_beg,
1520
+ :tIDENTIFIER, "b", :expr_arg)
1394
1521
  end
1395
1522
 
1396
1523
  def test_yylex_lt_equals
1397
- util_lex_token "<=", :tLEQ, "<="
1524
+ assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
1398
1525
  end
1399
1526
 
1400
1527
  def test_yylex_minus
1401
- util_lex_token("1 - 2",
1402
- :tINTEGER, 1,
1403
- :tMINUS, "-",
1404
- :tINTEGER, 2)
1528
+ assert_lex3("1 - 2",
1529
+ nil,
1530
+ :tINTEGER, 1, :expr_end,
1531
+ :tMINUS, "-", :expr_beg,
1532
+ :tINTEGER, 2, :expr_end)
1405
1533
  end
1406
1534
 
1407
1535
  def test_yylex_minus_equals
1408
- util_lex_token "-=", :tOP_ASGN, "-"
1536
+ assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
1409
1537
  end
1410
1538
 
1411
1539
  def test_yylex_minus_method
1412
- @lex.lex_state = :expr_fname
1413
- util_lex_token "-", :tMINUS, "-"
1540
+ self.lex_state = :expr_fname
1541
+
1542
+ assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
1414
1543
  end
1415
1544
 
1416
1545
  def test_yylex_minus_unary_method
1417
- @lex.lex_state = :expr_fname
1418
- util_lex_token "-@", :tUMINUS, "-@"
1546
+ self.lex_state = :expr_fname
1547
+
1548
+ assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
1419
1549
  end
1420
1550
 
1421
1551
  def test_yylex_minus_unary_number
1422
- util_lex_token("-42",
1423
- :tUMINUS_NUM, "-",
1424
- :tINTEGER, 42)
1552
+ assert_lex3("-42",
1553
+ nil,
1554
+ :tUMINUS_NUM, "-", :expr_beg,
1555
+ :tINTEGER, 42, :expr_end)
1425
1556
  end
1426
1557
 
1427
1558
  def test_yylex_nth_ref
1428
- util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
1429
- :tLBRACK, "[",
1430
- :tNTH_REF, 1, :tCOMMA, ",",
1431
- :tNTH_REF, 2, :tCOMMA, ",",
1432
- :tNTH_REF, 3, :tCOMMA, ",",
1433
- :tNTH_REF, 4, :tCOMMA, ",",
1434
- :tNTH_REF, 5, :tCOMMA, ",",
1435
- :tNTH_REF, 6, :tCOMMA, ",",
1436
- :tNTH_REF, 7, :tCOMMA, ",",
1437
- :tNTH_REF, 8, :tCOMMA, ",",
1438
- :tNTH_REF, 9,
1439
- :tRBRACK, "]")
1559
+ assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
1560
+ nil,
1561
+ :tLBRACK, "[", :expr_beg,
1562
+ :tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1563
+ :tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1564
+ :tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
1565
+ :tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
1566
+ :tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
1567
+ :tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
1568
+ :tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
1569
+ :tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
1570
+ :tNTH_REF, 9, :expr_end,
1571
+ :tRBRACK, "]", :expr_endarg)
1440
1572
  end
1441
1573
 
1442
1574
  def test_yylex_open_bracket
1443
- util_lex_token("(", :tLPAREN, "(")
1575
+ assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
1444
1576
  end
1445
1577
 
1446
1578
  def test_yylex_open_bracket_cmdarg
1447
- @lex.lex_state = :expr_cmdarg
1448
- util_lex_token(" (", :tLPAREN_ARG, "(")
1579
+ self.lex_state = :expr_cmdarg
1580
+
1581
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1449
1582
  end
1450
1583
 
1451
1584
  def test_yylex_open_bracket_exprarg__18
1452
1585
  setup_lexer_class Ruby18Parser
1586
+ self.lex_state = :expr_arg
1453
1587
 
1454
- @lex.lex_state = :expr_arg
1455
- util_lex_token(" (", :tLPAREN2, "(")
1588
+ assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
1456
1589
  end
1457
1590
 
1458
1591
  def test_yylex_open_bracket_exprarg__19
1459
1592
  setup_lexer_class Ruby19Parser
1593
+ self.lex_state = :expr_arg
1460
1594
 
1461
- @lex.lex_state = :expr_arg
1462
- util_lex_token(" (", :tLPAREN_ARG, "(")
1595
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1463
1596
  end
1464
1597
 
1465
1598
  def test_yylex_open_curly_bracket
1466
- util_lex_token("{",
1467
- :tLBRACE, "{")
1599
+ assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
1468
1600
  end
1469
1601
 
1470
1602
  def test_yylex_open_curly_bracket_arg
1471
- @lex.lex_state = :expr_arg
1472
- util_lex_token("m { 3 }",
1473
- :tIDENTIFIER, "m",
1474
- :tLCURLY, "{",
1475
- :tINTEGER, 3,
1476
- :tRCURLY, "}")
1603
+ self.lex_state = :expr_arg
1604
+
1605
+ assert_lex3("m { 3 }",
1606
+ nil,
1607
+ :tIDENTIFIER, "m", :expr_cmdarg,
1608
+ :tLCURLY, "{", :expr_beg,
1609
+ :tINTEGER, 3, :expr_end,
1610
+ :tRCURLY, "}", :expr_endarg)
1477
1611
  end
1478
1612
 
1479
1613
  def test_yylex_open_curly_bracket_block
1480
- @lex.lex_state = :expr_endarg # seen m(3)
1481
- util_lex_token("{ 4 }",
1482
- :tLBRACE_ARG, "{",
1483
- :tINTEGER, 4,
1484
- :tRCURLY, "}")
1614
+ self.lex_state = :expr_endarg # seen m(3)
1615
+
1616
+ assert_lex3("{ 4 }",
1617
+ nil,
1618
+ :tLBRACE_ARG, "{", :expr_beg,
1619
+ :tINTEGER, 4, :expr_end,
1620
+ :tRCURLY, "}", :expr_endarg)
1485
1621
  end
1486
1622
 
1487
1623
  def test_yylex_open_square_bracket_arg
1488
- @lex.lex_state = :expr_arg
1489
- util_lex_token("m [ 3 ]",
1490
- :tIDENTIFIER, "m",
1491
- :tLBRACK, "[",
1492
- :tINTEGER, 3,
1493
- :tRBRACK, "]")
1624
+ self.lex_state = :expr_arg
1625
+
1626
+ assert_lex3("m [ 3 ]",
1627
+ nil,
1628
+ :tIDENTIFIER, "m", :expr_cmdarg,
1629
+ :tLBRACK, "[", :expr_beg,
1630
+ :tINTEGER, 3, :expr_end,
1631
+ :tRBRACK, "]", :expr_endarg)
1494
1632
  end
1495
1633
 
1496
1634
  def test_yylex_open_square_bracket_ary
1497
- util_lex_token("[1, 2, 3]",
1498
- :tLBRACK, "[",
1499
- :tINTEGER, 1,
1500
- :tCOMMA, ",",
1501
- :tINTEGER, 2,
1502
- :tCOMMA, ",",
1503
- :tINTEGER, 3,
1504
- :tRBRACK, "]")
1635
+ assert_lex3("[1, 2, 3]",
1636
+ nil,
1637
+ :tLBRACK, "[", :expr_beg,
1638
+ :tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1639
+ :tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1640
+ :tINTEGER, 3, :expr_end,
1641
+ :tRBRACK, "]", :expr_endarg)
1505
1642
  end
1506
1643
 
1507
1644
  def test_yylex_open_square_bracket_meth
1508
- util_lex_token("m[3]",
1509
- :tIDENTIFIER, "m",
1510
- :tLBRACK2, "[",
1511
- :tINTEGER, 3,
1512
- :tRBRACK, "]")
1645
+ assert_lex3("m[3]",
1646
+ nil,
1647
+ :tIDENTIFIER, "m", :expr_cmdarg,
1648
+ :tLBRACK2, "[", :expr_beg,
1649
+ :tINTEGER, 3, :expr_end,
1650
+ :tRBRACK, "]", :expr_endarg)
1513
1651
  end
1514
1652
 
1515
1653
  def test_yylex_or
1516
- util_lex_token "|", :tPIPE, "|"
1654
+ assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
1517
1655
  end
1518
1656
 
1519
1657
  def test_yylex_or2
1520
- util_lex_token "||", :tOROP, "||"
1658
+ assert_lex3("||", nil, :tOROP, "||", :expr_beg)
1521
1659
  end
1522
1660
 
1523
1661
  def test_yylex_or2_equals
1524
- util_lex_token "||=", :tOP_ASGN, "||"
1662
+ assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
1525
1663
  end
1526
1664
 
1527
1665
  def test_yylex_or_equals
1528
- util_lex_token "|=", :tOP_ASGN, "|"
1666
+ assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
1529
1667
  end
1530
1668
 
1531
1669
  def test_yylex_percent
1532
- util_lex_token("a % 2",
1533
- :tIDENTIFIER, "a",
1534
- :tPERCENT, "%",
1535
- :tINTEGER, 2)
1670
+ assert_lex3("a % 2",
1671
+ nil,
1672
+ :tIDENTIFIER, "a", :expr_cmdarg,
1673
+ :tPERCENT, "%", :expr_beg,
1674
+ :tINTEGER, 2, :expr_end)
1536
1675
  end
1537
1676
 
1538
1677
  def test_yylex_percent_equals
1539
- util_lex_token("a %= 2",
1540
- :tIDENTIFIER, "a",
1541
- :tOP_ASGN, "%",
1542
- :tINTEGER, 2)
1678
+ assert_lex3("a %= 2",
1679
+ nil,
1680
+ :tIDENTIFIER, "a", :expr_cmdarg,
1681
+ :tOP_ASGN, "%", :expr_beg,
1682
+ :tINTEGER, 2, :expr_end)
1543
1683
  end
1544
1684
 
1545
1685
  def test_yylex_plus
1546
- util_lex_token("1 + 1", # TODO lex_state?
1547
- :tINTEGER, 1,
1548
- :tPLUS, "+",
1549
- :tINTEGER, 1)
1686
+ assert_lex3("1 + 1", # TODO lex_state?
1687
+ nil,
1688
+ :tINTEGER, 1, :expr_end,
1689
+ :tPLUS, "+", :expr_beg,
1690
+ :tINTEGER, 1, :expr_end)
1550
1691
  end
1551
1692
 
1552
1693
  def test_yylex_plus_equals
1553
- util_lex_token "+=", :tOP_ASGN, "+"
1694
+ assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
1554
1695
  end
1555
1696
 
1556
1697
  def test_yylex_plus_method
1557
- @lex.lex_state = :expr_fname
1558
- util_lex_token "+", :tPLUS, "+"
1698
+ self.lex_state = :expr_fname
1699
+
1700
+ assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
1559
1701
  end
1560
1702
 
1561
1703
  def test_yylex_plus_unary_method
1562
- @lex.lex_state = :expr_fname
1563
- util_lex_token "+@", :tUPLUS, "+@"
1704
+ self.lex_state = :expr_fname
1705
+
1706
+ assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
1564
1707
  end
1565
1708
 
1566
1709
  def test_yylex_not_unary_method
1567
- skip "not yet"
1710
+ self.lex_state = :expr_fname
1568
1711
 
1569
- @lex.lex_state = :expr_fname
1570
- util_lex_token "!@", :tUBANG, "!@"
1712
+ assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
1571
1713
  end
1572
1714
 
1573
1715
  def test_yylex_numbers
1574
- util_lex_token "0b10", :tINTEGER, 2
1575
- util_lex_token "0B10", :tINTEGER, 2
1716
+ assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
1717
+ assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
1576
1718
 
1577
- util_lex_token "0d10", :tINTEGER, 10
1578
- util_lex_token "0D10", :tINTEGER, 10
1719
+ assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
1720
+ assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
1579
1721
 
1580
- util_lex_token "0x10", :tINTEGER, 16
1581
- util_lex_token "0X10", :tINTEGER, 16
1722
+ assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
1723
+ assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
1582
1724
 
1583
- util_lex_token "0o10", :tINTEGER, 8
1584
- util_lex_token "0O10", :tINTEGER, 8
1585
- util_lex_token "0o", :tINTEGER, 0
1586
- util_lex_token "0O", :tINTEGER, 0
1725
+ assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
1726
+ assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
1587
1727
 
1588
- util_lex_token "0o", :tINTEGER, 0
1589
- util_lex_token "0O", :tINTEGER, 0
1728
+ assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
1729
+ assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
1590
1730
 
1591
- util_lex_token "0", :tINTEGER, 0
1731
+ assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
1592
1732
 
1593
- util_bad_token "0x"
1594
- util_bad_token "0X"
1595
- util_bad_token "0b"
1596
- util_bad_token "0B"
1597
- util_bad_token "0d"
1598
- util_bad_token "0D"
1733
+ refute_lex "0x"
1734
+ refute_lex "0X"
1735
+ refute_lex "0b"
1736
+ refute_lex "0B"
1737
+ refute_lex "0d"
1738
+ refute_lex "0D"
1599
1739
 
1600
- util_bad_token "08"
1601
- util_bad_token "09"
1602
- util_bad_token "0o8"
1603
- util_bad_token "0o9"
1604
- util_bad_token "0O8"
1605
- util_bad_token "0O9"
1740
+ refute_lex "08"
1741
+ refute_lex "09"
1742
+ refute_lex "0o8"
1743
+ refute_lex "0o9"
1744
+ refute_lex "0O8"
1745
+ refute_lex "0O9"
1606
1746
 
1607
- util_bad_token "1_e1"
1608
- util_bad_token "1_.1"
1609
- util_bad_token "1__1"
1747
+ refute_lex "1_e1"
1748
+ refute_lex "1_.1"
1749
+ refute_lex "1__1"
1610
1750
  end
1611
1751
 
1612
1752
  def test_yylex_plus_unary_number
1613
- util_lex_token("+42",
1614
- :tINTEGER, 42)
1753
+ assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
1615
1754
  end
1616
1755
 
1617
1756
  def test_yylex_question__18
1618
1757
  setup_lexer_class Ruby18Parser
1619
1758
 
1620
- util_lex_token "?*", :tINTEGER, 42
1759
+ assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
1621
1760
  end
1622
1761
 
1623
1762
  def test_yylex_question__19
1624
1763
  setup_lexer_class Ruby19Parser
1625
1764
 
1626
- util_lex_token "?*", :tSTRING, "*"
1765
+ assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
1627
1766
  end
1628
1767
 
1629
1768
  def test_yylex_question_bad_eos
1630
- util_bad_token "?"
1769
+ refute_lex "?"
1631
1770
  end
1632
1771
 
1633
1772
  def test_yylex_question_ws
1634
- util_lex_token "? ", :tEH, "?"
1635
- util_lex_token "?\n", :tEH, "?"
1636
- util_lex_token "?\t", :tEH, "?"
1637
- util_lex_token "?\v", :tEH, "?"
1638
- util_lex_token "?\r", :tEH, "?"
1639
- util_lex_token "?\f", :tEH, "?"
1773
+ assert_lex3("? ", nil, :tEH, "?", :expr_value)
1774
+ assert_lex3("?\n", nil, :tEH, "?", :expr_value)
1775
+ assert_lex3("?\t", nil, :tEH, "?", :expr_value)
1776
+ assert_lex3("?\v", nil, :tEH, "?", :expr_value)
1777
+ assert_lex3("?\r", nil, :tEH, "?", :expr_value)
1778
+ assert_lex3("?\f", nil, :tEH, "?", :expr_value)
1640
1779
  end
1641
1780
 
1642
1781
  def test_yylex_question_ws_backslashed__18
1643
1782
  setup_lexer_class Ruby18Parser
1644
1783
 
1645
- @lex.lex_state = :expr_beg
1646
- util_lex_token "?\\ ", :tINTEGER, 32
1647
- @lex.lex_state = :expr_beg
1648
- util_lex_token "?\\n", :tINTEGER, 10
1649
- @lex.lex_state = :expr_beg
1650
- util_lex_token "?\\t", :tINTEGER, 9
1651
- @lex.lex_state = :expr_beg
1652
- util_lex_token "?\\v", :tINTEGER, 11
1653
- @lex.lex_state = :expr_beg
1654
- util_lex_token "?\\r", :tINTEGER, 13
1655
- @lex.lex_state = :expr_beg
1656
- util_lex_token "?\\f", :tINTEGER, 12
1784
+ assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
1785
+ assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
1786
+ assert_lex3("?\\t", nil, :tINTEGER, 9, :expr_end)
1787
+ assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
1788
+ assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
1789
+ assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
1657
1790
  end
1658
1791
 
1659
1792
  def test_yylex_question_ws_backslashed__19
1660
1793
  setup_lexer_class Ruby19Parser
1661
1794
 
1662
- @lex.lex_state = :expr_beg
1663
- util_lex_token "?\\ ", :tSTRING, " "
1664
- @lex.lex_state = :expr_beg
1665
- util_lex_token "?\\n", :tSTRING, "\n"
1666
- @lex.lex_state = :expr_beg
1667
- util_lex_token "?\\t", :tSTRING, "\t"
1668
- @lex.lex_state = :expr_beg
1669
- util_lex_token "?\\v", :tSTRING, "\v"
1670
- @lex.lex_state = :expr_beg
1671
- util_lex_token "?\\r", :tSTRING, "\r"
1672
- @lex.lex_state = :expr_beg
1673
- util_lex_token "?\\f", :tSTRING, "\f"
1795
+ assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
1796
+ assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
1797
+ assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
1798
+ assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
1799
+ assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
1800
+ assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
1674
1801
  end
1675
1802
 
1676
1803
  def test_yylex_rbracket
1677
- util_lex_token "]", :tRBRACK, "]"
1804
+ assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
1678
1805
  end
1679
1806
 
1680
1807
  def test_yylex_rcurly
1681
- util_lex_token "}", :tRCURLY, "}"
1808
+ assert_lex3("}", nil, :tRCURLY, "}", :expr_endarg)
1682
1809
  end
1683
1810
 
1684
1811
  def test_yylex_regexp
1685
- util_lex_token("/regexp/",
1686
- :tREGEXP_BEG, "/",
1687
- :tSTRING_CONTENT, "regexp",
1688
- :tREGEXP_END, "")
1812
+ assert_lex3("/regexp/",
1813
+ nil,
1814
+ :tREGEXP_BEG, "/", :expr_beg,
1815
+ :tSTRING_CONTENT, "regexp", :expr_beg,
1816
+ :tREGEXP_END, "", :expr_end)
1689
1817
  end
1690
1818
 
1691
1819
  def test_yylex_regexp_ambiguous
1692
- util_lex_token("method /regexp/",
1693
- :tIDENTIFIER, "method",
1694
- :tREGEXP_BEG, "/",
1695
- :tSTRING_CONTENT, "regexp",
1696
- :tREGEXP_END, "")
1820
+ assert_lex3("method /regexp/",
1821
+ nil,
1822
+ :tIDENTIFIER, "method", :expr_cmdarg,
1823
+ :tREGEXP_BEG, "/", :expr_cmdarg,
1824
+ :tSTRING_CONTENT, "regexp", :expr_cmdarg,
1825
+ :tREGEXP_END, "", :expr_end)
1697
1826
  end
1698
1827
 
1699
1828
  def test_yylex_regexp_bad
1700
- util_bad_token("/.*/xyz",
1701
- :tREGEXP_BEG, "/",
1702
- :tSTRING_CONTENT, ".*")
1829
+ refute_lex("/.*/xyz",
1830
+ :tREGEXP_BEG, "/",
1831
+ :tSTRING_CONTENT, ".*")
1703
1832
  end
1704
1833
 
1705
1834
  def test_yylex_regexp_escape_C
1706
- util_lex_token('/regex\\C-x/',
1707
- :tREGEXP_BEG, "/",
1708
- :tSTRING_CONTENT, "regex\\C-x",
1709
- :tREGEXP_END, "")
1835
+ assert_lex3("/regex\\C-x/",
1836
+ nil,
1837
+ :tREGEXP_BEG, "/", :expr_beg,
1838
+ :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
1839
+ :tREGEXP_END, "", :expr_end)
1710
1840
  end
1711
1841
 
1712
1842
  def test_yylex_regexp_escape_C_M
1713
- util_lex_token('/regex\\C-\\M-x/',
1714
- :tREGEXP_BEG, "/",
1715
- :tSTRING_CONTENT, "regex\\C-\\M-x",
1716
- :tREGEXP_END, "")
1843
+ assert_lex3("/regex\\C-\\M-x/",
1844
+ nil,
1845
+ :tREGEXP_BEG, "/", :expr_beg,
1846
+ :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1847
+ :tREGEXP_END, "", :expr_end)
1717
1848
  end
1718
1849
 
1719
1850
  def test_yylex_regexp_escape_C_M_craaaazy
1720
- util_lex_token("/regex\\C-\\\n\\M-x/",
1721
- :tREGEXP_BEG, "/",
1722
- :tSTRING_CONTENT, "regex\\C-\\M-x",
1723
- :tREGEXP_END, "")
1851
+ assert_lex3("/regex\\C-\\\n\\M-x/",
1852
+ nil,
1853
+ :tREGEXP_BEG, "/", :expr_beg,
1854
+ :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1855
+ :tREGEXP_END, "", :expr_end)
1724
1856
  end
1725
1857
 
1726
1858
  def test_yylex_regexp_escape_C_bad_dash
1727
- util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
1859
+ refute_lex '/regex\\Cx/', :tREGEXP_BEG, "/"
1728
1860
  end
1729
1861
 
1730
1862
  def test_yylex_regexp_escape_C_bad_dash_eos
1731
- util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
1863
+ refute_lex '/regex\\C-/', :tREGEXP_BEG, "/"
1732
1864
  end
1733
1865
 
1734
1866
  def test_yylex_regexp_escape_C_bad_dash_eos2
1735
- util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
1867
+ refute_lex '/regex\\C-', :tREGEXP_BEG, "/"
1736
1868
  end
1737
1869
 
1738
1870
  def test_yylex_regexp_escape_C_bad_eos
1739
- util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
1871
+ refute_lex '/regex\\C/', :tREGEXP_BEG, "/"
1740
1872
  end
1741
1873
 
1742
1874
  def test_yylex_regexp_escape_C_bad_eos2
1743
- util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
1875
+ refute_lex '/regex\\c', :tREGEXP_BEG, "/"
1744
1876
  end
1745
1877
 
1746
1878
  def test_yylex_regexp_escape_M
1747
- util_lex_token('/regex\\M-x/',
1748
- :tREGEXP_BEG, "/",
1749
- :tSTRING_CONTENT, "regex\\M-x",
1750
- :tREGEXP_END, "")
1879
+ assert_lex3("/regex\\M-x/",
1880
+ nil,
1881
+ :tREGEXP_BEG, "/", :expr_beg,
1882
+ :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
1883
+ :tREGEXP_END, "", :expr_end)
1751
1884
  end
1752
1885
 
1753
1886
  def test_yylex_regexp_escape_M_C
1754
- util_lex_token('/regex\\M-\\C-x/',
1755
- :tREGEXP_BEG, "/",
1756
- :tSTRING_CONTENT, "regex\\M-\\C-x",
1757
- :tREGEXP_END, "")
1887
+ assert_lex3("/regex\\M-\\C-x/",
1888
+ nil,
1889
+ :tREGEXP_BEG, "/", :expr_beg,
1890
+ :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
1891
+ :tREGEXP_END, "", :expr_end)
1758
1892
  end
1759
1893
 
1760
1894
  def test_yylex_regexp_escape_M_bad_dash
1761
- util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
1895
+ refute_lex '/regex\\Mx/', :tREGEXP_BEG, "/"
1762
1896
  end
1763
1897
 
1764
1898
  def test_yylex_regexp_escape_M_bad_dash_eos
1765
- util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
1899
+ refute_lex '/regex\\M-/', :tREGEXP_BEG, "/"
1766
1900
  end
1767
1901
 
1768
1902
  def test_yylex_regexp_escape_M_bad_dash_eos2
1769
- util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
1903
+ refute_lex '/regex\\M-', :tREGEXP_BEG, "/"
1770
1904
  end
1771
1905
 
1772
1906
  def test_yylex_regexp_escape_M_bad_eos
1773
- util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
1907
+ refute_lex '/regex\\M/', :tREGEXP_BEG, "/"
1774
1908
  end
1775
1909
 
1776
1910
  def test_yylex_regexp_escape_backslash_slash
1777
- util_lex_token('/\\//',
1778
- :tREGEXP_BEG, "/",
1779
- :tSTRING_CONTENT, '\\/',
1780
- :tREGEXP_END, "")
1911
+ assert_lex3("/\\//",
1912
+ nil,
1913
+ :tREGEXP_BEG, "/", :expr_beg,
1914
+ :tSTRING_CONTENT, "\\/", :expr_beg,
1915
+ :tREGEXP_END, "", :expr_end)
1781
1916
  end
1782
1917
 
1783
1918
  def test_yylex_regexp_escape_backslash_terminator
1784
- util_lex_token('%r%blah\\%blah%',
1785
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1786
- :tSTRING_CONTENT, "blah\\%blah",
1787
- :tREGEXP_END, "")
1919
+ assert_lex3("%r%blah\\%blah%",
1920
+ nil,
1921
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1922
+ :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
1923
+ :tREGEXP_END, "", :expr_end)
1788
1924
  end
1789
1925
 
1790
1926
  def test_yylex_regexp_escape_backslash_terminator_meta1
1791
- util_lex_token('%r{blah\\}blah}',
1792
- :tREGEXP_BEG, "%r{", # FIX ?!?
1793
- :tSTRING_CONTENT, "blah\\}blah",
1794
- :tREGEXP_END, "")
1927
+ assert_lex3("%r{blah\\}blah}",
1928
+ nil,
1929
+ :tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
1930
+ :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
1931
+ :tREGEXP_END, "", :expr_end)
1795
1932
  end
1796
1933
 
1797
1934
  def test_yylex_regexp_escape_backslash_terminator_meta2
1798
- util_lex_token('%r/blah\\/blah/',
1799
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1800
- :tSTRING_CONTENT, "blah\\/blah",
1801
- :tREGEXP_END, "")
1935
+ assert_lex3("%r/blah\\/blah/",
1936
+ nil,
1937
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1938
+ :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
1939
+ :tREGEXP_END, "", :expr_end)
1802
1940
  end
1803
1941
 
1804
1942
  def test_yylex_regexp_escape_backslash_terminator_meta3
1805
- util_lex_token('%r/blah\\%blah/',
1806
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1807
- :tSTRING_CONTENT, "blah\\%blah",
1808
- :tREGEXP_END, "")
1943
+ assert_lex3("%r/blah\\%blah/",
1944
+ nil,
1945
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1946
+ :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
1947
+ :tREGEXP_END, "", :expr_end)
1809
1948
  end
1810
1949
 
1811
1950
  def test_yylex_regexp_escape_bad_eos
1812
- util_bad_token '/regex\\', :tREGEXP_BEG, "/"
1951
+ refute_lex '/regex\\', :tREGEXP_BEG, "/"
1813
1952
  end
1814
1953
 
1815
1954
  def test_yylex_regexp_escape_bs
1816
- util_lex_token('/regex\\\\regex/',
1817
- :tREGEXP_BEG, "/",
1818
- :tSTRING_CONTENT, "regex\\\\regex",
1819
- :tREGEXP_END, "")
1955
+ assert_lex3("/regex\\\\regex/",
1956
+ nil,
1957
+ :tREGEXP_BEG, "/", :expr_beg,
1958
+ :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
1959
+ :tREGEXP_END, "", :expr_end)
1820
1960
  end
1821
1961
 
1822
1962
  def test_yylex_regexp_escape_c
1823
- util_lex_token('/regex\\cxxx/',
1824
- :tREGEXP_BEG, "/",
1825
- :tSTRING_CONTENT, "regex\\cxxx",
1826
- :tREGEXP_END, "")
1963
+ assert_lex3("/regex\\cxxx/",
1964
+ nil,
1965
+ :tREGEXP_BEG, "/", :expr_beg,
1966
+ :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
1967
+ :tREGEXP_END, "", :expr_end)
1827
1968
  end
1828
1969
 
1829
1970
  def test_yylex_regexp_escape_c_backslash
1830
- util_lex_token('/regex\\c\\n/',
1831
- :tREGEXP_BEG, "/",
1832
- :tSTRING_CONTENT, "regex\\c\\n",
1833
- :tREGEXP_END, "")
1971
+ assert_lex3("/regex\\c\\n/",
1972
+ nil,
1973
+ :tREGEXP_BEG, "/", :expr_beg,
1974
+ :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
1975
+ :tREGEXP_END, "", :expr_end)
1834
1976
  end
1835
1977
 
1836
1978
  def test_yylex_regexp_escape_chars
1837
- util_lex_token('/re\\tge\\nxp/',
1838
- :tREGEXP_BEG, "/",
1839
- :tSTRING_CONTENT, "re\\tge\\nxp",
1840
- :tREGEXP_END, "")
1979
+ assert_lex3("/re\\tge\\nxp/",
1980
+ nil,
1981
+ :tREGEXP_BEG, "/", :expr_beg,
1982
+ :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
1983
+ :tREGEXP_END, "", :expr_end)
1841
1984
  end
1842
1985
 
1843
1986
  def test_yylex_regexp_escape_double_backslash
1844
1987
  regexp = '/[\\/\\\\]$/'
1845
- util_lex_token(regexp,
1846
- :tREGEXP_BEG, "/",
1847
- :tSTRING_CONTENT, regexp[1..-2],
1848
- :tREGEXP_END, "")
1988
+ assert_lex3(regexp.dup,
1989
+ nil,
1990
+ :tREGEXP_BEG, "/", :expr_beg,
1991
+ :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
1992
+ :tREGEXP_END, "", :expr_end)
1849
1993
  end
1850
1994
 
1851
1995
  def test_yylex_regexp_escape_hex
1852
- util_lex_token('/regex\\x61xp/',
1853
- :tREGEXP_BEG, "/",
1854
- :tSTRING_CONTENT, "regex\\x61xp",
1855
- :tREGEXP_END, "")
1996
+ assert_lex3("/regex\\x61xp/",
1997
+ nil,
1998
+ :tREGEXP_BEG, "/", :expr_beg,
1999
+ :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
2000
+ :tREGEXP_END, "", :expr_end)
1856
2001
  end
1857
2002
 
1858
2003
  def test_yylex_regexp_escape_hex_bad
1859
- util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
2004
+ refute_lex '/regex\\xzxp/', :tREGEXP_BEG, "/"
1860
2005
  end
1861
2006
 
1862
2007
  def test_yylex_regexp_escape_hex_one
1863
- util_lex_token('/^[\\xd\\xa]{2}/on',
1864
- :tREGEXP_BEG, '/',
1865
- :tSTRING_CONTENT, '^[\\xd\\xa]{2}',
1866
- :tREGEXP_END, 'on')
2008
+ assert_lex3("/^[\\xd\\xa]{2}/on",
2009
+ nil,
2010
+ :tREGEXP_BEG, "/", :expr_beg,
2011
+ :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
2012
+ :tREGEXP_END, "on", :expr_end)
1867
2013
  end
1868
2014
 
1869
2015
  def test_yylex_regexp_escape_oct1
1870
- util_lex_token('/regex\\0xp/',
1871
- :tREGEXP_BEG, "/",
1872
- :tSTRING_CONTENT, "regex\\0xp",
1873
- :tREGEXP_END, "")
2016
+ assert_lex3("/regex\\0xp/",
2017
+ nil,
2018
+ :tREGEXP_BEG, "/", :expr_beg,
2019
+ :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
2020
+ :tREGEXP_END, "", :expr_end)
1874
2021
  end
1875
2022
 
1876
2023
  def test_yylex_regexp_escape_oct2
1877
- util_lex_token('/regex\\07xp/',
1878
- :tREGEXP_BEG, "/",
1879
- :tSTRING_CONTENT, "regex\\07xp",
1880
- :tREGEXP_END, "")
2024
+ assert_lex3("/regex\\07xp/",
2025
+ nil,
2026
+ :tREGEXP_BEG, "/", :expr_beg,
2027
+ :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
2028
+ :tREGEXP_END, "", :expr_end)
1881
2029
  end
1882
2030
 
1883
2031
  def test_yylex_regexp_escape_oct3
1884
- util_lex_token('/regex\\10142/',
1885
- :tREGEXP_BEG, "/",
1886
- :tSTRING_CONTENT, "regex\\10142",
1887
- :tREGEXP_END, "")
2032
+ assert_lex3("/regex\\10142/",
2033
+ nil,
2034
+ :tREGEXP_BEG, "/", :expr_beg,
2035
+ :tSTRING_CONTENT, "regex\\10142", :expr_beg,
2036
+ :tREGEXP_END, "", :expr_end)
1888
2037
  end
1889
2038
 
1890
2039
  def test_yylex_regexp_escape_return
1891
- util_lex_token("/regex\\\nregex/",
1892
- :tREGEXP_BEG, "/",
1893
- :tSTRING_CONTENT, "regexregex",
1894
- :tREGEXP_END, "")
2040
+ assert_lex3("/regex\\\nregex/",
2041
+ nil,
2042
+ :tREGEXP_BEG, "/", :expr_beg,
2043
+ :tSTRING_CONTENT, "regexregex", :expr_beg,
2044
+ :tREGEXP_END, "", :expr_end)
1895
2045
  end
1896
2046
 
1897
2047
  def test_yylex_regexp_nm
1898
- util_lex_token("/.*/nm",
1899
- :tREGEXP_BEG, "/",
1900
- :tSTRING_CONTENT, ".*",
1901
- :tREGEXP_END, "nm")
2048
+ assert_lex3("/.*/nm",
2049
+ nil,
2050
+ :tREGEXP_BEG, "/", :expr_beg,
2051
+ :tSTRING_CONTENT, ".*", :expr_beg,
2052
+ :tREGEXP_END, "nm", :expr_end)
1902
2053
  end
1903
2054
 
1904
2055
  def test_yylex_rparen
1905
- util_lex_token ")", :tRPAREN, ")"
2056
+ assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
1906
2057
  end
1907
2058
 
1908
2059
  def test_yylex_rshft
1909
- util_lex_token("a >> 2",
1910
- :tIDENTIFIER, "a",
1911
- :tRSHFT, ">>",
1912
- :tINTEGER, 2)
2060
+ assert_lex3("a >> 2",
2061
+ nil,
2062
+ :tIDENTIFIER, "a", :expr_cmdarg,
2063
+ :tRSHFT, ">>", :expr_beg,
2064
+ :tINTEGER, 2, :expr_end)
1913
2065
  end
1914
2066
 
1915
2067
  def test_yylex_rshft_equals
1916
- util_lex_token("a >>= 2",
1917
- :tIDENTIFIER, "a",
1918
- :tOP_ASGN, ">>",
1919
- :tINTEGER, 2)
2068
+ assert_lex3("a >>= 2",
2069
+ nil,
2070
+ :tIDENTIFIER, "a", :expr_cmdarg,
2071
+ :tOP_ASGN, ">>", :expr_beg,
2072
+ :tINTEGER, 2, :expr_end)
1920
2073
  end
1921
2074
 
1922
2075
  def test_yylex_star
1923
- util_lex_token("a * ",
1924
- :tIDENTIFIER, "a",
1925
- :tSTAR2, "*")
1926
-
1927
- assert_equal :expr_beg, @lex.lex_state
2076
+ assert_lex3("a * ",
2077
+ nil,
2078
+ :tIDENTIFIER, "a", :expr_cmdarg,
2079
+ :tSTAR2, "*", :expr_beg)
1928
2080
  end
1929
2081
 
1930
2082
  def test_yylex_star2
1931
- util_lex_token("a ** ",
1932
- :tIDENTIFIER, "a",
1933
- :tPOW, "**")
1934
-
1935
- assert_equal :expr_beg, @lex.lex_state
2083
+ assert_lex3("a ** ",
2084
+ nil,
2085
+ :tIDENTIFIER, "a", :expr_cmdarg,
2086
+ :tPOW, "**", :expr_beg)
1936
2087
  end
1937
2088
 
1938
2089
  def test_yylex_star2_equals
1939
- util_lex_token("a **= ",
1940
- :tIDENTIFIER, "a",
1941
- :tOP_ASGN, "**")
1942
-
1943
- assert_equal :expr_beg, @lex.lex_state
2090
+ assert_lex3("a **= ",
2091
+ nil,
2092
+ :tIDENTIFIER, "a", :expr_cmdarg,
2093
+ :tOP_ASGN, "**", :expr_beg)
1944
2094
  end
1945
2095
 
1946
2096
  def test_yylex_star_arg
1947
- @lex.lex_state = :expr_arg
1948
-
1949
- util_lex_token(" *a",
1950
- :tSTAR, "*",
1951
- :tIDENTIFIER, "a")
2097
+ self.lex_state = :expr_arg
1952
2098
 
1953
- assert_equal :expr_arg, @lex.lex_state
2099
+ assert_lex3(" *a",
2100
+ nil,
2101
+ :tSTAR, "*", :expr_beg,
2102
+ :tIDENTIFIER, "a", :expr_arg)
1954
2103
  end
1955
2104
 
1956
2105
  def test_yylex_star_arg_beg
1957
- @lex.lex_state = :expr_beg
2106
+ self.lex_state = :expr_beg
1958
2107
 
1959
- util_lex_token("*a",
1960
- :tSTAR, "*",
1961
- :tIDENTIFIER, "a")
1962
-
1963
- assert_equal :expr_arg, @lex.lex_state
2108
+ assert_lex3("*a",
2109
+ nil,
2110
+ :tSTAR, "*", :expr_beg,
2111
+ :tIDENTIFIER, "a", :expr_arg)
1964
2112
  end
1965
2113
 
1966
2114
  def test_yylex_star_arg_beg_fname
1967
- @lex.lex_state = :expr_fname
2115
+ self.lex_state = :expr_fname
2116
+
2117
+ assert_lex3("*a",
2118
+ nil,
2119
+ :tSTAR2, "*", :expr_arg,
2120
+ :tIDENTIFIER, "a", :expr_arg)
2121
+ end
1968
2122
 
1969
- util_lex_token("*a",
1970
- :tSTAR2, "*",
1971
- :tIDENTIFIER, "a")
2123
+ def test_yylex_star_arg_beg_fname2
2124
+ self.lex_state = :expr_fname
1972
2125
 
1973
- assert_equal :expr_arg, @lex.lex_state
2126
+ assert_lex3("*a",
2127
+ nil,
2128
+ :tSTAR2, "*", :expr_arg,
2129
+ :tIDENTIFIER, "a", :expr_arg)
1974
2130
  end
1975
2131
 
1976
2132
  def test_yylex_star_equals
1977
- util_lex_token("a *= ",
1978
- :tIDENTIFIER, "a",
1979
- :tOP_ASGN, "*")
1980
-
1981
- assert_equal :expr_beg, @lex.lex_state
2133
+ assert_lex3("a *= ",
2134
+ nil,
2135
+ :tIDENTIFIER, "a", :expr_cmdarg,
2136
+ :tOP_ASGN, "*", :expr_beg)
1982
2137
  end
1983
2138
 
1984
2139
  def test_yylex_string_bad_eos
1985
- util_bad_token('%',
1986
- :tSTRING_BEG, '%')
2140
+ refute_lex('%', :tSTRING_BEG, '%')
1987
2141
  end
1988
2142
 
1989
2143
  def test_yylex_string_bad_eos_quote
1990
- util_bad_token('%{nest',
1991
- :tSTRING_BEG, '%}')
2144
+ refute_lex('%{nest', :tSTRING_BEG, '%}')
1992
2145
  end
1993
2146
 
1994
2147
  def test_yylex_string_double
1995
- util_lex_token('"string"',
1996
- :tSTRING, "string")
2148
+ assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
1997
2149
  end
1998
2150
 
1999
2151
  def test_yylex_string_double_escape_C
2000
- util_lex_token('"\\C-a"',
2001
- :tSTRING, "\001")
2152
+ assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
2002
2153
  end
2003
2154
 
2004
2155
  def test_yylex_string_double_escape_C_backslash
2005
- util_lex_token('"\\C-\\\\"',
2006
- :tSTRING_BEG, "\"",
2007
- :tSTRING_CONTENT, "\034",
2008
- :tSTRING_END, "\"")
2156
+ assert_lex3("\"\\C-\\\\\"",
2157
+ nil,
2158
+ :tSTRING_BEG, "\"", :expr_beg,
2159
+ :tSTRING_CONTENT, "\034", :expr_beg,
2160
+ :tSTRING_END, "\"", :expr_end)
2009
2161
  end
2010
2162
 
2011
2163
  def test_yylex_string_double_escape_C_escape
2012
- util_lex_token('"\\C-\\M-a"',
2013
- :tSTRING_BEG, "\"",
2014
- :tSTRING_CONTENT, "\201",
2015
- :tSTRING_END, "\"")
2164
+ assert_lex3("\"\\C-\\M-a\"",
2165
+ nil,
2166
+ :tSTRING_BEG, "\"", :expr_beg,
2167
+ :tSTRING_CONTENT, "\201", :expr_beg,
2168
+ :tSTRING_END, "\"", :expr_end)
2016
2169
  end
2017
2170
 
2018
2171
  def test_yylex_string_double_escape_C_question
2019
- util_lex_token('"\\C-?"',
2020
- :tSTRING, "\177")
2172
+ assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
2173
+ end
2174
+
2175
+ def test_yylex_string_utf8_simple
2176
+ chr = [0x3024].pack("U")
2177
+
2178
+ assert_lex3('"\u{3024}"',
2179
+ s(:str, chr),
2180
+ :tSTRING, chr, :expr_end)
2181
+ end
2182
+
2183
+ def test_yylex_string_utf8_complex
2184
+ chr = [0x3024].pack("U")
2185
+
2186
+ assert_lex3('"#@a\u{3024}"',
2187
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2188
+ :tSTRING_BEG, '"', :expr_beg,
2189
+ :tSTRING_DVAR, nil, :expr_beg,
2190
+ :tSTRING_CONTENT, "@a"+chr, :expr_beg,
2191
+ :tSTRING_END, '"', :expr_end)
2021
2192
  end
2022
2193
 
2023
2194
  def test_yylex_string_double_escape_M
2024
2195
  chr = "\341"
2025
2196
  chr.force_encoding("UTF-8") if RubyLexer::RUBY19
2026
2197
 
2027
- util_lex_token('"\\M-a"',
2028
- :tSTRING, chr)
2198
+ assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
2029
2199
  end
2030
2200
 
2031
2201
  def test_why_does_ruby_hate_me?
2032
- util_lex_token('"Nl%\000\000A\000\999"', # you should be ashamed
2033
- :tSTRING,
2034
- ["Nl%", "\x00", "\x00", "A", "\x00", "999"].join)
2202
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2203
+ nil,
2204
+ :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
2035
2205
  end
2036
2206
 
2037
2207
  def test_yylex_string_double_escape_M_backslash
2038
- util_lex_token('"\\M-\\\\"',
2039
- :tSTRING_BEG, "\"",
2040
- :tSTRING_CONTENT, "\334",
2041
- :tSTRING_END, "\"")
2208
+ assert_lex3("\"\\M-\\\\\"",
2209
+ nil,
2210
+ :tSTRING_BEG, "\"", :expr_beg,
2211
+ :tSTRING_CONTENT, "\334", :expr_beg,
2212
+ :tSTRING_END, "\"", :expr_end)
2042
2213
  end
2043
2214
 
2044
2215
  def test_yylex_string_double_escape_M_escape
2045
- util_lex_token('"\\M-\\C-a"',
2046
- :tSTRING_BEG, "\"",
2047
- :tSTRING_CONTENT, "\201",
2048
- :tSTRING_END, "\"")
2216
+ assert_lex3("\"\\M-\\C-a\"",
2217
+ nil,
2218
+ :tSTRING_BEG, "\"", :expr_beg,
2219
+ :tSTRING_CONTENT, "\201", :expr_beg,
2220
+ :tSTRING_END, "\"", :expr_end)
2049
2221
  end
2050
2222
 
2051
2223
  def test_yylex_string_double_escape_bs1
2052
- util_lex_token('"a\\a\\a"',
2053
- :tSTRING, "a\a\a")
2224
+ assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
2054
2225
  end
2055
2226
 
2056
2227
  def test_yylex_string_double_escape_bs2
2057
- util_lex_token('"a\\\\a"',
2058
- :tSTRING, "a\\a")
2228
+ assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
2059
2229
  end
2060
2230
 
2061
2231
  def test_yylex_string_double_escape_c
2062
- util_lex_token('"\\ca"',
2063
- :tSTRING, "\001")
2232
+ assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
2064
2233
  end
2065
2234
 
2066
2235
  def test_yylex_string_double_escape_c_backslash
2067
- util_lex_token('"\\c\\"',
2068
- :tSTRING_BEG, "\"",
2069
- :tSTRING_CONTENT, "\034",
2070
- :tSTRING_END, "\"")
2236
+ assert_lex3("\"\\c\\\"",
2237
+ nil,
2238
+ :tSTRING_BEG, "\"", :expr_beg,
2239
+ :tSTRING_CONTENT, "\034", :expr_beg,
2240
+ :tSTRING_END, "\"", :expr_end)
2071
2241
  end
2072
2242
 
2073
2243
  def test_yylex_string_double_escape_c_escape
2074
- util_lex_token('"\\c\\M-a"',
2075
- :tSTRING_BEG, "\"",
2076
- :tSTRING_CONTENT, "\201",
2077
- :tSTRING_END, "\"")
2244
+ assert_lex3("\"\\c\\M-a\"",
2245
+ nil,
2246
+ :tSTRING_BEG, "\"", :expr_beg,
2247
+ :tSTRING_CONTENT, "\201", :expr_beg,
2248
+ :tSTRING_END, "\"", :expr_end)
2078
2249
  end
2079
2250
 
2080
2251
  def test_yylex_string_double_escape_c_question
2081
- util_lex_token('"\\c?"',
2082
- :tSTRING, "\177")
2252
+ assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
2083
2253
  end
2084
2254
 
2085
2255
  def test_yylex_string_double_escape_chars
2086
- util_lex_token('"s\\tri\\ng"',
2087
- :tSTRING, "s\tri\ng")
2256
+ assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
2088
2257
  end
2089
2258
 
2090
2259
  def test_yylex_string_double_escape_hex
2091
- util_lex_token('"n = \\x61\\x62\\x63"',
2092
- :tSTRING, "n = abc")
2260
+ assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
2093
2261
  end
2094
2262
 
2095
2263
  def test_yylex_string_double_escape_octal
2096
- util_lex_token('"n = \\101\\102\\103"',
2097
- :tSTRING, "n = ABC")
2264
+ assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
2098
2265
  end
2099
2266
 
2100
2267
  def test_yylex_string_double_escape_octal_fucked
2101
- util_lex_token('"n = \\444"',
2102
- :tSTRING, "n = $")
2268
+ assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
2103
2269
  end
2104
2270
 
2105
2271
  def test_yylex_string_double_interp
2106
- util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
2107
- :tSTRING_BEG, "\"",
2108
- :tSTRING_CONTENT, "blah #x a ",
2109
- :tSTRING_DVAR, nil,
2110
- :tSTRING_CONTENT, "@a b ",
2111
- :tSTRING_DVAR, nil,
2112
- :tSTRING_CONTENT, "$b c ",
2113
- :tSTRING_DBEG, nil,
2114
- :tSTRING_CONTENT, "3} # ",
2115
- :tSTRING_END, "\"")
2272
+ assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
2273
+ nil,
2274
+ :tSTRING_BEG, "\"", :expr_beg,
2275
+ :tSTRING_CONTENT, "blah #x a ", :expr_beg,
2276
+ :tSTRING_DVAR, nil, :expr_beg,
2277
+ :tSTRING_CONTENT, "@a b ", :expr_beg,
2278
+ :tSTRING_DVAR, nil, :expr_beg,
2279
+ :tSTRING_CONTENT, "$b c ", :expr_beg,
2280
+ :tSTRING_DBEG, nil, :expr_beg,
2281
+ :tSTRING_CONTENT, "3} # ", :expr_beg,
2282
+ :tSTRING_END, "\"", :expr_end)
2116
2283
  end
2117
2284
 
2118
2285
  def test_yylex_string_double_nested_curlies
2119
- util_lex_token('%{nest{one{two}one}nest}',
2120
- :tSTRING_BEG, '%}',
2121
- :tSTRING_CONTENT, "nest{one{two}one}nest",
2122
- :tSTRING_END, '}')
2286
+ assert_lex3("%{nest{one{two}one}nest}",
2287
+ nil,
2288
+ :tSTRING_BEG, "%}", :expr_beg,
2289
+ :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
2290
+ :tSTRING_END, "}", :expr_end)
2123
2291
  end
2124
2292
 
2125
2293
  def test_yylex_string_double_no_interp
2126
- util_lex_token("\"# blah\"", # pound first
2127
- :tSTRING, "# blah")
2128
-
2129
- util_lex_token("\"blah # blah\"", # pound not first
2130
- :tSTRING, "blah # blah")
2294
+ assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
2295
+ assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
2131
2296
  end
2132
2297
 
2133
2298
  def test_yylex_string_escape_x_single
2134
- util_lex_token('"\\x0"',
2135
- :tSTRING, "\000")
2299
+ assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
2136
2300
  end
2137
2301
 
2138
2302
  def test_yylex_string_pct_i
2139
- util_lex_token("%i[s1 s2\ns3]",
2140
- :tQSYMBOLS_BEG, "%i[",
2141
- :tSTRING_CONTENT, "s1",
2142
- :tSPACE, nil,
2143
- :tSTRING_CONTENT, "s2",
2144
- :tSPACE, nil,
2145
- :tSTRING_CONTENT, "s3",
2146
- :tSPACE, nil,
2147
- :tSTRING_END, nil)
2303
+ assert_lex3("%i[s1 s2\ns3]",
2304
+ nil,
2305
+ :tQSYMBOLS_BEG, "%i[", :expr_beg,
2306
+ :tSTRING_CONTENT, "s1", :expr_beg,
2307
+ :tSPACE, nil, :expr_beg,
2308
+ :tSTRING_CONTENT, "s2", :expr_beg,
2309
+ :tSPACE, nil, :expr_beg,
2310
+ :tSTRING_CONTENT, "s3", :expr_beg,
2311
+ :tSPACE, nil, :expr_beg,
2312
+ :tSTRING_END, nil, :expr_end)
2148
2313
  end
2149
2314
 
2150
2315
  def test_yylex_string_pct_I
2151
- util_lex_token("%I[s1 s2\ns3]",
2152
- :tSYMBOLS_BEG, "%I[",
2153
- :tSTRING_CONTENT, "s1",
2154
- :tSPACE, nil,
2155
- :tSTRING_CONTENT, "s2",
2156
- :tSPACE, nil,
2157
- :tSTRING_CONTENT, "s3",
2158
- :tSPACE, nil,
2159
- :tSTRING_END, nil)
2316
+ assert_lex3("%I[s1 s2\ns3]",
2317
+ nil,
2318
+ :tSYMBOLS_BEG, "%I[", :expr_beg,
2319
+ :tSTRING_CONTENT, "s1", :expr_beg,
2320
+ :tSPACE, nil, :expr_beg,
2321
+ :tSTRING_CONTENT, "s2", :expr_beg,
2322
+ :tSPACE, nil, :expr_beg,
2323
+ :tSTRING_CONTENT, "s3", :expr_beg,
2324
+ :tSPACE, nil, :expr_beg,
2325
+ :tSTRING_END, nil, :expr_end)
2326
+ end
2327
+
2328
+ def test_yylex_string_pct_i_extra_space
2329
+ assert_lex3("%i[ s1 s2\ns3 ]",
2330
+ nil,
2331
+ :tQSYMBOLS_BEG, "%i[", :expr_beg,
2332
+ :tSTRING_CONTENT, "s1", :expr_beg,
2333
+ :tSPACE, nil, :expr_beg,
2334
+ :tSTRING_CONTENT, "s2", :expr_beg,
2335
+ :tSPACE, nil, :expr_beg,
2336
+ :tSTRING_CONTENT, "s3", :expr_beg,
2337
+ :tSPACE, nil, :expr_beg,
2338
+ :tSTRING_END, nil, :expr_end)
2339
+ end
2340
+
2341
+ def test_yylex_string_pct_I_extra_space
2342
+ assert_lex3("%I[ s1 s2\ns3 ]",
2343
+ nil,
2344
+ :tSYMBOLS_BEG, "%I[", :expr_beg,
2345
+ :tSTRING_CONTENT, "s1", :expr_beg,
2346
+ :tSPACE, nil, :expr_beg,
2347
+ :tSTRING_CONTENT, "s2", :expr_beg,
2348
+ :tSPACE, nil, :expr_beg,
2349
+ :tSTRING_CONTENT, "s3", :expr_beg,
2350
+ :tSPACE, nil, :expr_beg,
2351
+ :tSTRING_END, nil, :expr_end)
2352
+ end
2353
+
2354
+ def test_yylex_string_pct_q
2355
+ assert_lex3("%q[s1 s2]",
2356
+ nil,
2357
+ :tSTRING_BEG, "%q[", :expr_beg,
2358
+ :tSTRING_CONTENT, "s1 s2", :expr_beg,
2359
+ :tSTRING_END, "]", :expr_end)
2160
2360
  end
2161
2361
 
2162
2362
  def test_yylex_string_pct_Q
2163
- util_lex_token("%Q[s1 s2]",
2164
- :tSTRING_BEG, "%Q[",
2165
- :tSTRING_CONTENT, "s1 s2",
2166
- :tSTRING_END, "]")
2363
+ assert_lex3("%Q[s1 s2]",
2364
+ nil,
2365
+ :tSTRING_BEG, "%Q[", :expr_beg,
2366
+ :tSTRING_CONTENT, "s1 s2", :expr_beg,
2367
+ :tSTRING_END, "]", :expr_end)
2167
2368
  end
2168
2369
 
2169
2370
  def test_yylex_string_pct_W
2170
- util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
2171
- :tWORDS_BEG, "%W[",
2172
- :tSTRING_CONTENT, "s1",
2173
- :tSPACE, nil,
2174
- :tSTRING_CONTENT, "s2",
2175
- :tSPACE, nil,
2176
- :tSTRING_CONTENT, "s3",
2177
- :tSPACE, nil,
2178
- :tSTRING_END, nil)
2371
+ assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2372
+ nil,
2373
+ :tWORDS_BEG, "%W[", :expr_beg,
2374
+ :tSTRING_CONTENT, "s1", :expr_beg,
2375
+ :tSPACE, nil, :expr_beg,
2376
+ :tSTRING_CONTENT, "s2", :expr_beg,
2377
+ :tSPACE, nil, :expr_beg,
2378
+ :tSTRING_CONTENT, "s3", :expr_beg,
2379
+ :tSPACE, nil, :expr_beg,
2380
+ :tSTRING_END, nil, :expr_end)
2179
2381
  end
2180
2382
 
2181
2383
  def test_yylex_string_pct_W_bs_nl
2182
- util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
2183
- :tWORDS_BEG, "%W[",
2184
- :tSTRING_CONTENT, "s1",
2185
- :tSPACE, nil,
2186
- :tSTRING_CONTENT, "\ns2",
2187
- :tSPACE, nil,
2188
- :tSTRING_END, nil)
2384
+ assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2385
+ nil,
2386
+ :tWORDS_BEG, "%W[", :expr_beg,
2387
+ :tSTRING_CONTENT, "s1", :expr_beg,
2388
+ :tSPACE, nil, :expr_beg,
2389
+ :tSTRING_CONTENT, "\ns2", :expr_beg,
2390
+ :tSPACE, nil, :expr_beg,
2391
+ :tSTRING_END, nil, :expr_end)
2189
2392
  end
2190
2393
 
2191
2394
  def test_yylex_string_pct_angle
2192
- util_lex_token("%<blah>",
2193
- :tSTRING_BEG, "%>",
2194
- :tSTRING_CONTENT, "blah",
2195
- :tSTRING_END, ">")
2395
+ assert_lex3("%<blah>",
2396
+ nil,
2397
+ :tSTRING_BEG, "%>", :expr_beg,
2398
+ :tSTRING_CONTENT, "blah", :expr_beg,
2399
+ :tSTRING_END, ">", :expr_end)
2196
2400
  end
2197
2401
 
2198
2402
  def test_yylex_string_pct_other
2199
- util_lex_token("%%blah%",
2200
- :tSTRING_BEG, "%%",
2201
- :tSTRING_CONTENT, "blah",
2202
- :tSTRING_END, "%")
2403
+ assert_lex3("%%blah%",
2404
+ nil,
2405
+ :tSTRING_BEG, "%%", :expr_beg,
2406
+ :tSTRING_CONTENT, "blah", :expr_beg,
2407
+ :tSTRING_END, "%", :expr_end)
2203
2408
  end
2204
2409
 
2205
2410
  def test_yylex_string_pct_w
2206
- util_bad_token("%w[s1 s2 ",
2207
- :tQWORDS_BEG, "%w[",
2208
- :tSTRING_CONTENT, "s1",
2209
- :tSPACE, nil,
2210
- :tSTRING_CONTENT, "s2",
2211
- :tSPACE, nil)
2411
+ refute_lex("%w[s1 s2 ",
2412
+ :tQWORDS_BEG, "%w[",
2413
+ :tSTRING_CONTENT, "s1",
2414
+ :tSPACE, nil,
2415
+ :tSTRING_CONTENT, "s2",
2416
+ :tSPACE, nil)
2212
2417
  end
2213
2418
 
2214
2419
  def test_yylex_string_pct_w_bs_nl
2215
- util_lex_token("%w[s1 \\\ns2]",
2216
- :tQWORDS_BEG, "%w[",
2217
- :tSTRING_CONTENT, "s1",
2218
- :tSPACE, nil,
2219
- :tSTRING_CONTENT, "\ns2",
2220
- :tSPACE, nil,
2221
- :tSTRING_END, nil)
2420
+ assert_lex3("%w[s1 \\\ns2]",
2421
+ nil,
2422
+ :tQWORDS_BEG, "%w[", :expr_beg,
2423
+ :tSTRING_CONTENT, "s1", :expr_beg,
2424
+ :tSPACE, nil, :expr_beg,
2425
+ :tSTRING_CONTENT, "\ns2", :expr_beg,
2426
+ :tSPACE, nil, :expr_beg,
2427
+ :tSTRING_END, nil, :expr_end)
2222
2428
  end
2223
2429
 
2224
2430
  def test_yylex_string_pct_w_bs_sp
2225
- util_lex_token("%w[s\\ 1 s\\ 2]",
2226
- :tQWORDS_BEG, "%w[",
2227
- :tSTRING_CONTENT, "s 1",
2228
- :tSPACE, nil,
2229
- :tSTRING_CONTENT, "s 2",
2230
- :tSPACE, nil,
2231
- :tSTRING_END, nil)
2431
+ assert_lex3("%w[s\\ 1 s\\ 2]",
2432
+ nil,
2433
+ :tQWORDS_BEG, "%w[", :expr_beg,
2434
+ :tSTRING_CONTENT, "s 1", :expr_beg,
2435
+ :tSPACE, nil, :expr_beg,
2436
+ :tSTRING_CONTENT, "s 2", :expr_beg,
2437
+ :tSPACE, nil, :expr_beg,
2438
+ :tSTRING_END, nil, :expr_end)
2232
2439
  end
2233
2440
 
2234
2441
  def test_yylex_string_single
2235
- util_lex_token("'string'",
2236
- :tSTRING, "string")
2442
+ assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
2237
2443
  end
2238
2444
 
2239
2445
  def test_yylex_string_single_escape_chars
2240
- util_lex_token("'s\\tri\\ng'",
2241
- :tSTRING, "s\\tri\\ng")
2446
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
2242
2447
  end
2243
2448
 
2244
2449
  def test_yylex_string_single_nl
2245
- util_lex_token("'blah\\\nblah'",
2246
- :tSTRING, "blah\\\nblah")
2450
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
2247
2451
  end
2248
2452
 
2249
2453
  def test_yylex_symbol
2250
- util_lex_token(":symbol",
2251
- :tSYMBOL, "symbol")
2454
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
2252
2455
  end
2253
2456
 
2254
- def test_yylex_symbol_bad_zero
2255
- util_bad_token(":\"blah\0\"",
2256
- :tSYMBEG, ":")
2457
+ def test_yylex_symbol_zero_byte__18
2458
+ setup_lexer_class Ruby18Parser
2459
+
2460
+ refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
2461
+ end
2462
+
2463
+ def test_yylex_symbol_zero_byte
2464
+ assert_lex(":\"symbol\0\"", nil,
2465
+ :tSYMBOL, "symbol\0", :expr_end)
2257
2466
  end
2258
2467
 
2259
2468
  def test_yylex_symbol_double
2260
- util_lex_token(":\"symbol\"",
2261
- :tSYMBEG, ":",
2262
- :tSTRING_CONTENT, "symbol",
2263
- :tSTRING_END, '"')
2469
+ assert_lex3(":\"symbol\"",
2470
+ nil,
2471
+ :tSYMBOL, "symbol", :expr_end)
2472
+ end
2473
+
2474
+ def test_yylex_symbol_double_interp
2475
+ assert_lex3(':"symbol#{1+1}"',
2476
+ nil,
2477
+ :tSYMBEG, ":", :expr_fname,
2478
+ :tSTRING_CONTENT, "symbol", :expr_fname,
2479
+ :tSTRING_DBEG, nil, :expr_fname,
2480
+ :tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
2481
+ :tSTRING_END, "\"", :expr_end)
2264
2482
  end
2265
2483
 
2266
2484
  def test_yylex_symbol_single
2267
- util_lex_token(":'symbol'",
2268
- :tSYMBEG, ":",
2269
- :tSTRING_CONTENT, "symbol",
2270
- :tSTRING_END, "'")
2485
+ assert_lex3(":'symbol'",
2486
+ nil,
2487
+ :tSYMBOL, "symbol", :expr_end)
2271
2488
  end
2272
2489
 
2273
- def test_yylex_ternary
2274
- util_lex_token("a ? b : c",
2275
- :tIDENTIFIER, "a",
2276
- :tEH, "?",
2277
- :tIDENTIFIER, "b",
2278
- :tCOLON, ":",
2279
- :tIDENTIFIER, "c")
2490
+ def test_yylex_symbol_single_noninterp
2491
+ assert_lex3(':\'symbol#{1+1}\'',
2492
+ nil,
2493
+ :tSYMBOL, 'symbol#{1+1}', :expr_end)
2494
+ end
2280
2495
 
2281
- util_lex_token("a ?bb : c", # GAH! MATZ!!!
2282
- :tIDENTIFIER, "a",
2283
- :tEH, "?",
2284
- :tIDENTIFIER, "bb",
2285
- :tCOLON, ":",
2286
- :tIDENTIFIER, "c")
2496
+ def test_yylex_ternary1
2497
+ assert_lex3("a ? b : c",
2498
+ nil,
2499
+ :tIDENTIFIER, "a", :expr_cmdarg,
2500
+ :tEH, "?", :expr_value,
2501
+ :tIDENTIFIER, "b", :expr_arg,
2502
+ :tCOLON, ":", :expr_beg,
2503
+ :tIDENTIFIER, "c", :expr_arg)
2504
+
2505
+ assert_lex3("a ?bb : c", # GAH! MATZ!!!
2506
+ nil,
2507
+ :tIDENTIFIER, "a", :expr_cmdarg,
2508
+ :tEH, "?", :expr_beg,
2509
+ :tIDENTIFIER, "bb", :expr_arg,
2510
+ :tCOLON, ":", :expr_beg,
2511
+ :tIDENTIFIER, "c", :expr_arg)
2287
2512
 
2288
- util_lex_token("42 ?", # 42 forces expr_end
2289
- :tINTEGER, 42,
2290
- :tEH, "?")
2513
+ assert_lex3("42 ?",
2514
+ nil,
2515
+ :tINTEGER, 42, :expr_end,
2516
+ :tEH, "?", :expr_value)
2291
2517
  end
2292
2518
 
2293
2519
  def test_yylex_tilde
2294
- util_lex_token "~", :tTILDE, "~"
2520
+ assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
2295
2521
  end
2296
2522
 
2297
2523
  def test_yylex_tilde_unary
2298
- @lex.lex_state = :expr_fname
2299
- util_lex_token "~@", :tTILDE, "~"
2524
+ self.lex_state = :expr_fname
2525
+
2526
+ assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
2300
2527
  end
2301
2528
 
2302
2529
  def test_yylex_uminus
2303
- util_lex_token("-blah",
2304
- :tUMINUS, "-",
2305
- :tIDENTIFIER, "blah")
2530
+ assert_lex3("-blah",
2531
+ nil,
2532
+ :tUMINUS, "-", :expr_beg,
2533
+ :tIDENTIFIER, "blah", :expr_arg)
2306
2534
  end
2307
2535
 
2308
2536
  def test_yylex_underscore
2309
- util_lex_token("_var", :tIDENTIFIER, "_var")
2537
+ assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
2310
2538
  end
2311
2539
 
2312
2540
  def test_yylex_underscore_end
2313
2541
  @lex.src = "__END__\n"
2314
- refute @lex.advance
2542
+ refute_lexeme
2315
2543
  end
2316
2544
 
2317
2545
  def test_yylex_uplus
2318
- util_lex_token("+blah",
2319
- :tUPLUS, "+",
2320
- :tIDENTIFIER, "blah")
2546
+ assert_lex3("+blah",
2547
+ nil,
2548
+ :tUPLUS, "+", :expr_beg,
2549
+ :tIDENTIFIER, "blah", :expr_arg)
2321
2550
  end
2322
2551
 
2323
2552
  def test_zbug_float_in_decl
2324
- util_lex_token("def initialize(u = ",
2325
- :kDEF, "def",
2326
- :tIDENTIFIER, "initialize",
2327
- :tLPAREN2, "(",
2328
- :tIDENTIFIER, "u",
2329
- :tEQL, "=")
2330
-
2331
- assert_equal :expr_beg, @lex.lex_state
2332
-
2333
- util_lex_token("0.0, s = 0.0",
2334
- :tFLOAT, 0.0,
2335
- :tCOMMA, ',',
2336
- :tIDENTIFIER, "s",
2337
- :tEQL, "=",
2338
- :tFLOAT, 0.0)
2553
+ assert_lex3("def initialize(u = 0.0, s = 0.0",
2554
+ nil,
2555
+ :kDEF, "def", :expr_fname,
2556
+ :tIDENTIFIER, "initialize", :expr_endfn,
2557
+ :tLPAREN2, "(", :expr_beg,
2558
+ :tIDENTIFIER, "u", :expr_arg,
2559
+ :tEQL, "=", :expr_beg,
2560
+ :tFLOAT, 0.0, :expr_end,
2561
+ :tCOMMA, ",", :expr_beg,
2562
+ :tIDENTIFIER, "s", :expr_arg,
2563
+ :tEQL, "=", :expr_beg,
2564
+ :tFLOAT, 0.0, :expr_end)
2339
2565
  end
2340
2566
 
2341
2567
  def test_zbug_id_equals
2342
- util_lex_token("a =",
2343
- :tIDENTIFIER, "a",
2344
- :tEQL, "=")
2345
-
2346
- assert_equal :expr_beg, @lex.lex_state
2347
-
2348
- util_lex_token("0.0",
2349
- :tFLOAT, 0.0)
2568
+ assert_lex3("a = 0.0",
2569
+ nil,
2570
+ :tIDENTIFIER, "a", :expr_cmdarg,
2571
+ :tEQL, "=", :expr_beg,
2572
+ :tFLOAT, 0.0, :expr_end)
2350
2573
  end
2351
2574
 
2352
2575
  def test_zbug_no_spaces_in_decl
2353
- util_lex_token("def initialize(u=",
2354
- :kDEF, "def",
2355
- :tIDENTIFIER, "initialize",
2356
- :tLPAREN2, "(",
2357
- :tIDENTIFIER, "u",
2358
- :tEQL, "=")
2359
-
2360
- assert_equal :expr_beg, @lex.lex_state
2361
-
2362
- util_lex_token("0.0,s=0.0",
2363
- :tFLOAT, 0.0,
2364
- :tCOMMA, ",",
2365
- :tIDENTIFIER, "s",
2366
- :tEQL, "=",
2367
- :tFLOAT, 0.0)
2576
+ assert_lex3("def initialize(u=0.0,s=0.0",
2577
+ nil,
2578
+ :kDEF, "def", :expr_fname,
2579
+ :tIDENTIFIER, "initialize", :expr_endfn,
2580
+ :tLPAREN2, "(", :expr_beg,
2581
+ :tIDENTIFIER, "u", :expr_arg,
2582
+ :tEQL, "=", :expr_beg,
2583
+ :tFLOAT, 0.0, :expr_end,
2584
+ :tCOMMA, ",", :expr_beg,
2585
+ :tIDENTIFIER, "s", :expr_arg,
2586
+ :tEQL, "=", :expr_beg,
2587
+ :tFLOAT, 0.0, :expr_end)
2368
2588
  end
2369
2589
 
2370
2590
  def test_pct_w_backslashes
@@ -2381,53 +2601,5 @@ class TestRubyLexer < Minitest::Test
2381
2601
  :tSPACE, nil, :expr_beg, 0, 0,
2382
2602
  :tSTRING_END, nil, :expr_end, 0, 0)
2383
2603
  end
2384
-
2385
- # flunk "Not yet"
2386
- end
2387
-
2388
- ############################################################
2389
-
2390
- def util_bad_token s, *args
2391
- assert_raises RubyParser::SyntaxError do
2392
- util_lex_token s, *args
2393
- end
2394
- end
2395
-
2396
- def util_escape expected, input
2397
- @lex.src = input
2398
- assert_equal expected, @lex.read_escape, input
2399
- end
2400
-
2401
- def util_escape_bad input
2402
- @lex.src = input
2403
- assert_raises RubyParser::SyntaxError do
2404
- @lex.read_escape
2405
- end
2406
- end
2407
-
2408
- def util_lex_fname name, type, end_state = :expr_arg
2409
- @lex.lex_state = :expr_fname # can only set via parser's defs
2410
-
2411
- assert_lex("def #{name} ",
2412
- nil,
2413
-
2414
- :kDEF, "def", :expr_fname, 0, 0,
2415
- type, name, end_state, 0, 0)
2416
-
2417
- assert_equal end_state, @lex.lex_state
2418
- end
2419
-
2420
- def util_lex_token input, *args
2421
- @lex.src = input
2422
-
2423
- until args.empty? do
2424
- token = args.shift
2425
- value = args.shift
2426
- assert @lex.advance, "no more tokens"
2427
- # assert_equal [token, value].map(&:encoding), [@lex.token, [@lex.yacc_value].flatten.first].map(&:encoding), input # TODO
2428
- assert_equal [token, value], [@lex.token, [@lex.yacc_value].flatten.first], input
2429
- end
2430
-
2431
- refute @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
2432
2604
  end
2433
2605
  end