ruby_parser 3.2.2 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (19) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.autotest +0 -2
  5. data/History.txt +38 -0
  6. data/Rakefile +2 -6
  7. data/lib/ruby18_parser.rb +2003 -2010
  8. data/lib/ruby18_parser.y +6 -8
  9. data/lib/ruby19_parser.rb +2016 -2004
  10. data/lib/ruby19_parser.y +6 -8
  11. data/lib/ruby20_parser.rb +2189 -2154
  12. data/lib/ruby20_parser.y +21 -13
  13. data/lib/ruby_lexer.rb +645 -812
  14. data/lib/ruby_parser_extras.rb +17 -46
  15. data/test/test_ruby_lexer.rb +1280 -1108
  16. data/test/test_ruby_parser.rb +101 -1
  17. data/test/test_ruby_parser_extras.rb +1 -63
  18. metadata +112 -133
  19. metadata.gz.sig +0 -0
data/lib/ruby_parser_extras.rb CHANGED
@@ -111,7 +111,7 @@ class RPStringScanner < StringScanner
111
111
  end
112
112
 
113
113
  module RubyParserStuff
114
- VERSION = "3.2.2" unless constants.include? "VERSION" # SIGH
114
+ VERSION = "3.3.0" unless constants.include? "VERSION" # SIGH
115
115
 
116
116
  attr_accessor :lexer, :in_def, :in_single, :file
117
117
  attr_reader :env, :comments
@@ -282,7 +282,7 @@ module RubyParserStuff
282
282
 
283
283
  def aryset receiver, index
284
284
  index ||= []
285
- s(:attrasgn, receiver, :"[]=", *index[1..-1])
285
+ s(:attrasgn, receiver, :"[]=", *index[1..-1]).compact # [][1..-1] => nil
286
286
  end
287
287
 
288
288
  def assignable(lhs, value = nil)
@@ -306,25 +306,14 @@ module RubyParserStuff
306
306
  s(:cdecl, id)
307
307
  else
308
308
  case self.env[id]
309
- when :lvar then
309
+ when :lvar, :dvar, nil then
310
310
  s(:lasgn, id)
311
- when :dvar, nil then
312
- if self.env.current[id] == :dvar then
313
- s(:lasgn, id)
314
- elsif self.env[id] == :dvar then
315
- self.env.use(id)
316
- s(:lasgn, id)
317
- elsif ! self.env.dynamic? then
318
- s(:lasgn, id)
319
- else
320
- s(:lasgn, id)
321
- end
322
311
  else
323
312
  raise "wtf? unknown type: #{self.env[id]}"
324
313
  end
325
314
  end
326
315
 
327
- self.env[id] ||= :lvar unless result.sexp_type == :cdecl # HACK? cdecl
316
+ self.env[id] ||= :lvar if result.sexp_type == :lasgn
328
317
 
329
318
  result << value if value
330
319
 
@@ -443,9 +432,11 @@ module RubyParserStuff
443
432
  def initialize(options = {})
444
433
  super()
445
434
 
446
- v = self.class.name[/1[89]/]
435
+ v = self.class.name[/1[89]|20/]
436
+
447
437
  self.lexer = RubyLexer.new v && v.to_i
448
438
  self.lexer.parser = self
439
+
449
440
  @env = RubyParserStuff::Environment.new
450
441
  @comments = []
451
442
 
@@ -562,7 +553,8 @@ module RubyParserStuff
562
553
 
563
554
  if elsebody and not resbody then
564
555
  warning("else without rescue is useless")
565
- result = block_append(s(:begin, result), elsebody)
556
+ result = s(:begin, result) if result
557
+ result = block_append(result, elsebody)
566
558
  end
567
559
 
568
560
  result = s(:ensure, result, ensurebody).compact if ensurebody
@@ -668,8 +660,10 @@ module RubyParserStuff
668
660
  end
669
661
  end
670
662
 
663
+ args.line line
671
664
  result.line = line
672
665
  result.comments = self.comments.pop
666
+
673
667
  result
674
668
  end
675
669
 
@@ -786,7 +780,9 @@ module RubyParserStuff
786
780
  }[c]
787
781
  raise "unknown regexp option: #{c}" unless v
788
782
  o += v
789
- k = c if c =~ /[esu]/
783
+
784
+ # encoding options are ignored on 1.9+
785
+ k = c if c =~ /[esu]/ if RUBY_VERSION < "1.9"
790
786
  end
791
787
 
792
788
  case node[0]
@@ -938,7 +934,9 @@ module RubyParserStuff
938
934
  case lhs[0]
939
935
  when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
940
936
  lhs << rhs
941
- when :attrasgn, :call then
937
+ when :attrasgn then
938
+ lhs << rhs
939
+ when :call then
942
940
  args = lhs.pop unless Symbol === lhs.last
943
941
  lhs.concat arg_add(args, rhs)[1..-1]
944
942
  when :const then
@@ -1236,55 +1234,28 @@ module RubyParserStuff
1236
1234
  @env.first
1237
1235
  end
1238
1236
 
1239
- def dynamic
1240
- idx = @dyn.index false
1241
- @env[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
1242
- end
1243
-
1244
- def dynamic?
1245
- @dyn[0] != false
1246
- end
1247
-
1248
1237
  def extend dyn = false
1249
1238
  @dyn.unshift dyn
1250
1239
  @env.unshift({})
1251
- @use.unshift({})
1252
1240
  end
1253
1241
 
1254
1242
  def initialize dyn = false
1255
1243
  @dyn = []
1256
1244
  @env = []
1257
- @use = []
1258
1245
  self.reset
1259
1246
  end
1260
1247
 
1261
1248
  def reset
1262
1249
  @dyn.clear
1263
1250
  @env.clear
1264
- @use.clear
1265
1251
  self.extend
1266
1252
  end
1267
1253
 
1268
1254
  def unextend
1269
1255
  @dyn.shift
1270
1256
  @env.shift
1271
- @use.shift
1272
1257
  raise "You went too far unextending env" if @env.empty?
1273
1258
  end
1274
-
1275
- def use id
1276
- @env.each_with_index do |env, i|
1277
- if env[id] then
1278
- @use[i][id] = true
1279
- end
1280
- end
1281
- end
1282
-
1283
- def used? id
1284
- idx = @dyn.index false # REFACTOR
1285
- u = @use[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
1286
- u[id]
1287
- end
1288
1259
  end
1289
1260
 
1290
1261
  class StackState
data/test/test_ruby_lexer.rb CHANGED
@@ -8,20 +8,20 @@ require 'ruby18_parser'
8
8
  require 'ruby20_parser'
9
9
 
10
10
  class TestRubyLexer < Minitest::Test
11
- attr_accessor :processor, :lex, :parser_class
11
+ attr_accessor :processor, :lex, :parser_class, :lex_state
12
12
 
13
13
  alias :lexer :lex # lets me copy/paste code from parser
14
14
  alias :lexer= :lex=
15
15
 
16
16
  def setup
17
+ self.lex_state = :expr_beg
17
18
  setup_lexer_class Ruby20Parser
18
19
  end
19
20
 
20
21
  def setup_lexer input, exp_sexp = nil
21
22
  setup_new_parser
22
23
  lex.src = input
23
- lex.lex_state = :expr_beg
24
- assert_equal exp_sexp, processor.class.new.parse(input) if exp_sexp
24
+ lex.lex_state = self.lex_state
25
25
  end
26
26
 
27
27
  def setup_new_parser
@@ -35,6 +35,117 @@ class TestRubyLexer < Minitest::Test
35
35
  setup_lexer "blah blah"
36
36
  end
37
37
 
38
+ def assert_lex input, exp_sexp, *args, &b
39
+ setup_lexer input
40
+ assert_parse input, exp_sexp if exp_sexp
41
+
42
+ b.call if b
43
+
44
+ args.each_slice(5) do |token, value, state, paren, brace|
45
+ assert_next_lexeme token, value, state, paren, brace
46
+ end
47
+
48
+ refute_lexeme
49
+ end
50
+
51
+ def assert_lex3 input, exp_sexp, *args, &block
52
+ args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
53
+
54
+ assert_lex(input, exp_sexp, *args, &block)
55
+ end
56
+
57
+ def refute_lex input, *args # TODO: re-sort
58
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
59
+
60
+ assert_raises RubyParser::SyntaxError do
61
+ assert_lex(input, nil, *args)
62
+ end
63
+ end
64
+
65
+ def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
66
+ assert_lex3("def #{name} ",
67
+ nil,
68
+
69
+ :kDEF, "def", :expr_fname,
70
+ type, name, end_state)
71
+ end
72
+
73
+ def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
74
+ adv = @lex.advance
75
+
76
+ assert adv, "no more tokens"
77
+
78
+ act_token, act_value = @lex.token, @lex.yacc_value
79
+
80
+ msg = message {
81
+ act = [act_token, act_value, @lex.lex_state,
82
+ @lex.paren_nest, @lex.brace_nest]
83
+ exp = [token, value, state, paren, brace]
84
+ "#{exp.inspect} vs #{act.inspect}"
85
+ }
86
+
87
+ act_value = act_value.first if Array === act_value
88
+
89
+ assert_equal token, act_token, msg
90
+ assert_equal value, act_value, msg
91
+ assert_equal state, @lex.lex_state, msg if state
92
+ assert_equal paren, @lex.paren_nest, msg if paren
93
+ assert_equal brace, @lex.brace_nest, msg if brace
94
+ end
95
+
96
+ def assert_parse input, exp_sexp
97
+ assert_equal exp_sexp, processor.class.new.parse(input)
98
+ end
99
+
100
+ def assert_read_escape expected, input
101
+ @lex.src = input
102
+ assert_equal expected, @lex.read_escape, input
103
+ end
104
+
105
+ def assert_read_escape_bad input # TODO: rename refute_read_escape
106
+ @lex.src = input
107
+ assert_raises RubyParser::SyntaxError do
108
+ @lex.read_escape
109
+ end
110
+ end
111
+
112
+ def refute_lexeme
113
+ x = @lex.advance
114
+ y = [@lex.token, @lex.yacc_value]
115
+ refute x, "not empty: #{y.inspect}"
116
+ end
117
+
118
+ ## Utility Methods:
119
+
120
+ def emulate_string_interpolation
121
+ lex_strterm = lexer.lex_strterm
122
+ string_nest = lexer.string_nest
123
+ brace_nest = lexer.brace_nest
124
+
125
+ lexer.string_nest = 0
126
+ lexer.brace_nest = 0
127
+ lexer.cond.push false
128
+ lexer.cmdarg.push false
129
+
130
+ lexer.lex_strterm = nil
131
+ lexer.lex_state = :expr_beg
132
+
133
+ yield
134
+
135
+ lexer.lex_state = :expr_endarg
136
+ assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
137
+
138
+ lexer.lex_strterm = lex_strterm
139
+ lexer.lex_state = :expr_beg
140
+ lexer.string_nest = string_nest
141
+ lexer.brace_nest = brace_nest
142
+
143
+ lexer.cond.lexpop
144
+ lexer.cmdarg.lexpop
145
+ end
146
+
147
+ ## Tests:
148
+
38
149
  def test_advance
39
150
  assert @lex.advance # blah
40
151
  assert @lex.advance # blah
@@ -43,214 +154,162 @@ class TestRubyLexer < Minitest::Test
43
154
 
44
155
  def test_unicode_ident
45
156
  s = "@\u1088\u1077\u1093\u1072"
46
- util_lex_token(s.dup,
47
- :tIVAR, s.dup)
157
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
48
158
  end
49
159
 
50
160
  def test_read_escape
51
- util_escape "\\", "\\"
52
- util_escape "\n", "n"
53
- util_escape "\t", "t"
54
- util_escape "\r", "r"
55
- util_escape "\f", "f"
56
- util_escape "\13", "v"
57
- util_escape "\0", "0"
58
- util_escape "\07", "a"
59
- util_escape "\007", "a"
60
- util_escape "\033", "e"
61
- util_escape "\377", "377"
62
- util_escape "\377", "xff"
63
- util_escape "\010", "b"
64
- util_escape " ", "s"
65
- util_escape "q", "q" # plain vanilla escape
66
-
67
- util_escape "8", "8" # ugh... mri... WHY?!?
68
- util_escape "9", "9" # ugh... mri... WHY?!?
69
-
70
- util_escape "$", "444" # ugh
161
+ assert_read_escape "\\", "\\"
162
+ assert_read_escape "\n", "n"
163
+ assert_read_escape "\t", "t"
164
+ assert_read_escape "\r", "r"
165
+ assert_read_escape "\f", "f"
166
+ assert_read_escape "\13", "v"
167
+ assert_read_escape "\0", "0"
168
+ assert_read_escape "\07", "a"
169
+ assert_read_escape "\007", "a"
170
+ assert_read_escape "\033", "e"
171
+ assert_read_escape "\377", "377"
172
+ assert_read_escape "\377", "xff"
173
+ assert_read_escape "\010", "b"
174
+ assert_read_escape " ", "s"
175
+ assert_read_escape "q", "q" # plain vanilla escape
176
+
177
+ assert_read_escape "8", "8" # ugh... mri... WHY?!?
178
+ assert_read_escape "9", "9" # ugh... mri... WHY?!?
179
+
180
+ assert_read_escape "$", "444" # ugh
71
181
  end
72
182
 
73
183
  def test_read_escape_c
74
- util_escape "\030", "C-x"
75
- util_escape "\030", "cx"
76
- util_escape "\230", 'C-\M-x'
77
- util_escape "\230", 'c\M-x'
184
+ assert_read_escape "\030", "C-x"
185
+ assert_read_escape "\030", "cx"
186
+ assert_read_escape "\230", 'C-\M-x'
187
+ assert_read_escape "\230", 'c\M-x'
78
188
 
79
- util_escape "\177", "C-?"
80
- util_escape "\177", "c?"
189
+ assert_read_escape "\177", "C-?"
190
+ assert_read_escape "\177", "c?"
81
191
  end
82
192
 
83
193
  def test_read_escape_errors
84
- util_escape_bad ""
194
+ assert_read_escape_bad ""
85
195
 
86
- util_escape_bad "M"
87
- util_escape_bad "M-"
88
- util_escape_bad "Mx"
196
+ assert_read_escape_bad "M"
197
+ assert_read_escape_bad "M-"
198
+ assert_read_escape_bad "Mx"
89
199
 
90
- util_escape_bad "Cx"
91
- util_escape_bad "C"
92
- util_escape_bad "C-"
200
+ assert_read_escape_bad "Cx"
201
+ assert_read_escape_bad "C"
202
+ assert_read_escape_bad "C-"
93
203
 
94
- util_escape_bad "c"
204
+ assert_read_escape_bad "c"
95
205
  end
96
206
 
97
207
  def test_read_escape_m
98
- util_escape "\370", "M-x"
99
- util_escape "\230", 'M-\C-x'
100
- util_escape "\230", 'M-\cx'
208
+ assert_read_escape "\370", "M-x"
209
+ assert_read_escape "\230", 'M-\C-x'
210
+ assert_read_escape "\230", 'M-\cx'
101
211
  end
102
212
 
103
213
  def test_yylex_ambiguous_uminus
104
- util_lex_token("m -3",
105
- :tIDENTIFIER, "m",
106
- :tUMINUS_NUM, "-",
107
- :tINTEGER, 3)
214
+ assert_lex3("m -3",
215
+ nil,
216
+ :tIDENTIFIER, "m", :expr_cmdarg,
217
+ :tUMINUS_NUM, "-", :expr_beg,
218
+ :tINTEGER, 3, :expr_end)
219
+
108
220
  # TODO: verify warning
109
221
  end
110
222
 
111
223
  def test_yylex_ambiguous_uplus
112
- util_lex_token("m +3",
113
- :tIDENTIFIER, "m",
114
- :tINTEGER, 3)
224
+ assert_lex3("m +3",
225
+ nil,
226
+ :tIDENTIFIER, "m", :expr_cmdarg,
227
+ :tINTEGER, 3, :expr_end)
228
+
115
229
  # TODO: verify warning
116
230
  end
117
231
 
118
232
  def test_yylex_and
119
- util_lex_token "&", :tAMPER, "&"
233
+ assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
120
234
  end
121
235
 
122
236
  def test_yylex_and2
123
- util_lex_token "&&", :tANDOP, "&&"
237
+ assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
124
238
  end
125
239
 
126
240
  def test_yylex_and2_equals
127
- util_lex_token "&&=", :tOP_ASGN, "&&"
241
+ assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
128
242
  end
129
243
 
130
244
  def test_yylex_and_arg
131
- @lex.lex_state = :expr_arg
245
+ self.lex_state = :expr_arg
132
246
 
133
- util_lex_token(" &y",
134
- :tAMPER, "&",
135
- :tIDENTIFIER, "y")
247
+ assert_lex3(" &y",
248
+ nil,
249
+ :tAMPER, "&", :expr_beg,
250
+ :tIDENTIFIER, "y", :expr_arg)
136
251
  end
137
252
 
138
253
  def test_yylex_and_equals
139
- util_lex_token "&=", :tOP_ASGN, "&"
254
+ assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
140
255
  end
141
256
 
142
257
  def test_yylex_and_expr
143
- @lex.lex_state = :expr_arg
258
+ self.lex_state = :expr_arg
144
259
 
145
- util_lex_token("x & y",
146
- :tIDENTIFIER, "x",
147
- :tAMPER2, "&",
148
- :tIDENTIFIER, "y")
260
+ assert_lex3("x & y",
261
+ nil,
262
+ :tIDENTIFIER, "x", :expr_cmdarg,
263
+ :tAMPER2, "&", :expr_beg,
264
+ :tIDENTIFIER, "y", :expr_arg)
149
265
  end
150
266
 
151
267
  def test_yylex_and_meth
152
- util_lex_fname "&", :tAMPER2
268
+ assert_lex_fname "&", :tAMPER2
153
269
  end
154
270
 
155
271
  def test_yylex_assoc
156
- util_lex_token "=>", :tASSOC, "=>"
272
+ assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
157
273
  end
158
274
 
159
275
  def test_yylex_label__18
160
276
  setup_lexer_class Ruby18Parser
161
277
 
162
- util_lex_token("{a:",
163
- :tLBRACE, "{",
164
- :tIDENTIFIER, "a",
165
- :tSYMBEG, ":")
278
+ assert_lex3("{a:",
279
+ nil,
280
+ :tLBRACE, "{", :expr_beg,
281
+ :tIDENTIFIER, "a", :expr_arg,
282
+ :tSYMBEG, ":", :expr_fname)
166
283
  end
167
284
 
168
285
  def test_yylex_label_in_params__18
169
286
  setup_lexer_class Ruby18Parser
170
287
 
171
- util_lex_token("foo(a:",
172
- :tIDENTIFIER, "foo",
173
- :tLPAREN2, "(",
174
- :tIDENTIFIER, "a",
175
- :tSYMBEG, ":")
288
+ assert_lex3("foo(a:",
289
+ nil,
290
+ :tIDENTIFIER, "foo", :expr_cmdarg,
291
+ :tLPAREN2, "(", :expr_beg,
292
+ :tIDENTIFIER, "a", :expr_cmdarg,
293
+ :tSYMBEG, ":", :expr_fname)
176
294
  end
177
295
 
178
296
  def test_yylex_label__19
179
297
  setup_lexer_class Ruby19Parser
180
298
 
181
- util_lex_token("{a:",
182
- :tLBRACE, "{",
183
- :tLABEL, "a")
299
+ assert_lex3("{a:",
300
+ nil,
301
+ :tLBRACE, "{", :expr_beg,
302
+ :tLABEL, "a", :expr_beg)
184
303
  end
185
304
 
186
305
  def test_yylex_label_in_params__19
187
306
  setup_lexer_class Ruby19Parser
188
307
 
189
- util_lex_token("foo(a:",
190
- :tIDENTIFIER, "foo",
191
- :tLPAREN2, "(",
192
- :tLABEL, "a")
193
- end
194
-
195
- def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
196
- assert @lex.advance, "no more tokens"
197
-
198
- msg = message {
199
- act = [@lex.token, @lex.yacc_value, @lex.lex_state,
200
- @lex.paren_nest, @lex.brace_nest]
201
- exp = [token, value, state, paren, brace]
202
- "#{exp.inspect} vs #{act.inspect}"
203
- }
204
-
205
- act_value = @lex.yacc_value
206
- act_value = act_value.first if Array === act_value
207
-
208
- assert_equal token, @lex.token, msg
209
- assert_equal value, act_value, msg
210
- assert_equal state, @lex.lex_state, msg
211
- assert_equal paren, @lex.paren_nest, msg if paren
212
- assert_equal brace, @lex.brace_nest, msg if brace
213
- end
214
-
215
- def refute_lexeme
216
- refute @lex.advance, "not empty: #{[@lex.token, @lex.yacc_value].inspect}"
217
- end
218
-
219
- def assert_lex input, exp_sexp, *args
220
- setup_lexer input, exp_sexp
221
-
222
- args.each_slice(5) do |token, value, state, paren, brace|
223
- assert_next_lexeme token, value, state, paren, brace
224
- end
225
-
226
- refute_lexeme
227
- end
228
-
229
- def emulate_string_interpolation
230
- lex_strterm = lexer.lex_strterm
231
- string_nest = lexer.string_nest
232
- brace_nest = lexer.brace_nest
233
-
234
- lexer.string_nest = 0
235
- lexer.brace_nest = 0
236
- lexer.cond.push false
237
- lexer.cmdarg.push false
238
-
239
- lexer.lex_strterm = nil
240
- lexer.lex_state = :expr_beg
241
-
242
- yield
243
-
244
- lexer.lex_state = :expr_endarg
245
- assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
246
-
247
- lexer.lex_strterm = lex_strterm
248
- lexer.lex_state = :expr_beg
249
- lexer.string_nest = string_nest
250
- lexer.brace_nest = brace_nest
251
-
252
- lexer.cond.lexpop
253
- lexer.cmdarg.lexpop
308
+ assert_lex3("foo(a:",
309
+ nil,
310
+ :tIDENTIFIER, "foo", :expr_cmdarg,
311
+ :tLPAREN2, "(", :expr_beg,
312
+ :tLABEL, "a", :expr_beg)
254
313
  end
255
314
 
256
315
  def test_yylex_paren_string_parens_interpolated
@@ -532,1839 +591,2000 @@ class TestRubyLexer < Minitest::Test
532
591
  end
533
592
 
534
593
  def test_yylex_back_ref
535
- util_lex_token("[$&, $`, $', $+]",
536
- :tLBRACK, "[",
537
- :tBACK_REF, :"&", :tCOMMA, ",",
538
- :tBACK_REF, :"`", :tCOMMA, ",",
539
- :tBACK_REF, :"'", :tCOMMA, ",",
540
- :tBACK_REF, :"+",
541
- :tRBRACK, "]")
594
+ assert_lex3("[$&, $`, $', $+]",
595
+ nil,
596
+ :tLBRACK, "[", :expr_beg,
597
+ :tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
598
+ :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
599
+ :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
600
+ :tBACK_REF, :+, :expr_end,
601
+ :tRBRACK, "]", :expr_endarg)
542
602
  end
543
603
 
544
604
  def test_yylex_backslash
545
- util_lex_token("1 \\\n+ 2",
546
- :tINTEGER, 1,
547
- :tPLUS, "+",
548
- :tINTEGER, 2)
605
+ assert_lex3("1 \\\n+ 2",
606
+ nil,
607
+ :tINTEGER, 1, :expr_end,
608
+ :tPLUS, "+", :expr_beg,
609
+ :tINTEGER, 2, :expr_end)
549
610
  end
550
611
 
551
612
  def test_yylex_backslash_bad
552
- util_bad_token("1 \\ + 2",
553
- :tINTEGER, 1)
613
+ refute_lex("1 \\ + 2", :tINTEGER, 1)
554
614
  end
555
615
 
556
616
  def test_yylex_backtick
557
- util_lex_token("`ls`",
558
- :tXSTRING_BEG, "`",
559
- :tSTRING_CONTENT, "ls",
560
- :tSTRING_END, "`")
617
+ assert_lex3("`ls`",
618
+ nil,
619
+ :tXSTRING_BEG, "`", :expr_beg,
620
+ :tSTRING_CONTENT, "ls", :expr_beg,
621
+ :tSTRING_END, "`", :expr_end)
561
622
  end
562
623
 
563
624
  def test_yylex_backtick_cmdarg
564
- @lex.lex_state = :expr_dot
565
- util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
625
+ self.lex_state = :expr_dot
566
626
 
567
- assert_equal :expr_cmdarg, @lex.lex_state
627
+ # \n ensures expr_cmd (TODO: why?)
628
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
568
629
  end
569
630
 
570
631
  def test_yylex_backtick_dot
571
- @lex.lex_state = :expr_dot
572
- util_lex_token("a.`(3)",
573
- :tIDENTIFIER, "a",
574
- :tDOT, ".",
575
- :tBACK_REF2, "`",
576
- :tLPAREN2, "(",
577
- :tINTEGER, 3,
578
- :tRPAREN, ")")
632
+ self.lex_state = :expr_dot
633
+
634
+ assert_lex3("a.`(3)",
635
+ nil,
636
+ :tIDENTIFIER, "a", :expr_cmdarg,
637
+ :tDOT, ".", :expr_dot,
638
+ :tBACK_REF2, "`", :expr_arg,
639
+ :tLPAREN2, "(", :expr_beg,
640
+ :tINTEGER, 3, :expr_end,
641
+ :tRPAREN, ")", :expr_endfn)
579
642
  end
580
643
 
581
644
  def test_yylex_backtick_method
582
- @lex.lex_state = :expr_fname
583
- util_lex_token("`", :tBACK_REF2, "`")
584
- assert_equal :expr_end, @lex.lex_state
645
+ self.lex_state = :expr_fname
646
+
647
+ assert_lex3("`",
648
+ nil,
649
+ :tBACK_REF2, "`", :expr_end)
585
650
  end
586
651
 
587
652
  def test_yylex_bad_char
588
- util_bad_token(" \010 ")
653
+ refute_lex(" \010 ")
589
654
  end
590
655
 
591
656
  def test_yylex_bang
592
- util_lex_token "!", :tBANG, "!"
657
+ assert_lex3("!", nil, :tBANG, "!", :expr_beg)
593
658
  end
594
659
 
595
660
  def test_yylex_bang_equals
596
- util_lex_token "!=", :tNEQ, "!="
661
+ assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
597
662
  end
598
663
 
599
664
  def test_yylex_bang_tilde
600
- util_lex_token "!~", :tNMATCH, "!~"
665
+ assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
601
666
  end
602
667
 
603
668
  def test_yylex_carat
604
- util_lex_token "^", :tCARET, "^"
669
+ assert_lex3("^", nil, :tCARET, "^", :expr_beg)
605
670
  end
606
671
 
607
672
  def test_yylex_carat_equals
608
- util_lex_token "^=", :tOP_ASGN, "^"
673
+ assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
609
674
  end
610
675
 
611
676
  def test_yylex_colon2
612
- util_lex_token("A::B",
613
- :tCONSTANT, "A",
614
- :tCOLON2, "::",
615
- :tCONSTANT, "B")
677
+ assert_lex3("A::B",
678
+ nil,
679
+ :tCONSTANT, "A", :expr_cmdarg,
680
+ :tCOLON2, "::", :expr_dot,
681
+ :tCONSTANT, "B", :expr_arg)
682
+ end
683
+
684
+ def test_yylex_colon2_argh
685
+ assert_lex3("module X::Y\n c\nend",
686
+ nil,
687
+ :kMODULE, "module", :expr_value,
688
+ :tCONSTANT, "X", :expr_arg,
689
+ :tCOLON2, "::", :expr_dot,
690
+ :tCONSTANT, "Y", :expr_arg,
691
+ :tNL, nil, :expr_beg,
692
+ :tIDENTIFIER, "c", :expr_cmdarg,
693
+ :tNL, nil, :expr_beg,
694
+ :kEND, "end", :expr_end)
616
695
  end
617
696
 
618
697
  def test_yylex_colon3
619
- util_lex_token("::Array",
620
- :tCOLON3, "::",
621
- :tCONSTANT, "Array")
698
+ assert_lex3("::Array",
699
+ nil,
700
+ :tCOLON3, "::", :expr_beg,
701
+ :tCONSTANT, "Array", :expr_arg)
622
702
  end
623
703
 
624
704
  def test_yylex_comma
625
- util_lex_token ",", :tCOMMA, ","
705
+ assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
626
706
  end
627
707
 
628
708
  def test_yylex_comment
629
- util_lex_token("1 # one\n# two\n2",
630
- :tINTEGER, 1,
631
- :tNL, nil,
632
- :tINTEGER, 2)
709
+ assert_lex3("1 # one\n# two\n2",
710
+ nil,
711
+ :tINTEGER, 1, :expr_end,
712
+ :tNL, nil, :expr_beg,
713
+ :tINTEGER, 2, :expr_end)
714
+
633
715
  assert_equal "# one\n# two\n", @lex.comments
634
716
  end
635
717
 
636
718
  def test_yylex_comment_begin
637
- util_lex_token("=begin\nblah\nblah\n=end\n42",
638
- :tINTEGER, 42)
719
+ assert_lex3("=begin\nblah\nblah\n=end\n42",
720
+ nil,
721
+ :tINTEGER, 42, :expr_end)
722
+
639
723
  assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
640
724
  end
641
725
 
642
726
  def test_yylex_comment_begin_bad
643
- util_bad_token("=begin\nblah\nblah\n")
727
+ refute_lex("=begin\nblah\nblah\n")
728
+
644
729
  assert_equal "", @lex.comments
645
730
  end
646
731
 
647
732
  def test_yylex_comment_begin_not_comment
648
- util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
649
- :tIDENTIFIER, "beginfoo",
650
- :tEQL, "=",
651
- :tINTEGER, 5,
652
- :tNL, nil,
653
- :tIDENTIFIER, "p",
654
- :tIDENTIFIER, "x",
655
- :tEQL, "=",
656
- :tIDENTIFIER, "beginfoo")
733
+ assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
734
+ nil,
735
+ :tIDENTIFIER, "beginfoo", :expr_cmdarg,
736
+ :tEQL, "=", :expr_beg,
737
+ :tINTEGER, 5, :expr_end,
738
+ :tNL, nil, :expr_beg,
739
+ :tIDENTIFIER, "p", :expr_cmdarg,
740
+ :tIDENTIFIER, "x", :expr_arg,
741
+ :tEQL, "=", :expr_beg,
742
+ :tIDENTIFIER, "beginfoo", :expr_arg)
657
743
  end
658
744
 
659
745
  def test_yylex_comment_begin_space
660
- util_lex_token("=begin blah\nblah\n=end\n")
746
+ assert_lex3("=begin blah\nblah\n=end\n", nil)
747
+
661
748
  assert_equal "=begin blah\nblah\n=end\n", @lex.comments
662
749
  end
663
750
 
664
751
  def test_yylex_comment_end_space_and_text
665
- util_lex_token("=begin blah\nblah\n=end blab\n")
752
+ assert_lex3("=begin blah\nblah\n=end blab\n", nil)
753
+
666
754
  assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
667
755
  end
668
756
 
669
757
  def test_yylex_comment_eos
670
- util_lex_token("# comment")
758
+ assert_lex3("# comment", nil)
671
759
  end
672
760
 
673
761
  def test_yylex_constant
674
- util_lex_token("ArgumentError",
675
- :tCONSTANT, "ArgumentError")
762
+ assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
676
763
  end
677
764
 
678
765
  def test_yylex_constant_semi
679
- util_lex_token("ArgumentError;",
680
- :tCONSTANT, "ArgumentError",
681
- :tSEMI, ";")
766
+ assert_lex3("ArgumentError;",
767
+ nil,
768
+ :tCONSTANT, "ArgumentError", :expr_cmdarg,
769
+ :tSEMI, ";", :expr_beg)
682
770
  end
683
771
 
684
772
  def test_yylex_cvar
685
- util_lex_token "@@blah", :tCVAR, "@@blah"
773
+ assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
686
774
  end
687
775
 
688
776
  def test_yylex_cvar_bad
689
777
  assert_raises RubyParser::SyntaxError do
690
- util_lex_token "@@1"
778
+ assert_lex3("@@1", nil)
691
779
  end
692
780
  end
693
781
 
694
782
  def test_yylex_def_bad_name
695
- @lex.lex_state = :expr_fname
696
- util_bad_token("def [ ", :kDEF, "def")
783
+ self.lex_state = :expr_fname
784
+ refute_lex("def [ ", :kDEF, "def")
697
785
  end
698
786
 
699
787
  def test_yylex_div
700
- util_lex_token("a / 2",
701
- :tIDENTIFIER, "a",
702
- :tDIVIDE, "/",
703
- :tINTEGER, 2)
788
+ assert_lex3("a / 2",
789
+ nil,
790
+ :tIDENTIFIER, "a", :expr_cmdarg,
791
+ :tDIVIDE, "/", :expr_beg,
792
+ :tINTEGER, 2, :expr_end)
704
793
  end
705
794
 
706
795
  def test_yylex_div_equals
707
- util_lex_token("a /= 2",
708
- :tIDENTIFIER, "a",
709
- :tOP_ASGN, "/",
710
- :tINTEGER, 2)
796
+ assert_lex3("a /= 2",
797
+ nil,
798
+ :tIDENTIFIER, "a", :expr_cmdarg,
799
+ :tOP_ASGN, "/", :expr_beg,
800
+ :tINTEGER, 2, :expr_end)
711
801
  end
712
802
 
713
803
  def test_yylex_do
714
- util_lex_token("x do 42 end",
715
- :tIDENTIFIER, "x",
716
- :kDO, "do",
717
- :tINTEGER, 42,
718
- :kEND, "end")
804
+ assert_lex3("x do 42 end",
805
+ nil,
806
+ :tIDENTIFIER, "x", :expr_cmdarg,
807
+ :kDO, "do", :expr_beg,
808
+ :tINTEGER, 42, :expr_end,
809
+ :kEND, "end", :expr_end)
719
810
  end
720
811
 
721
812
  def test_yylex_do_block
722
- @lex.lex_state = :expr_endarg
723
- @lex.cmdarg.push true
724
-
725
- util_lex_token("x.y do 42 end",
726
- :tIDENTIFIER, "x",
727
- :tDOT, ".",
728
- :tIDENTIFIER, "y",
729
- :kDO_BLOCK, "do",
730
- :tINTEGER, 42,
731
- :kEND, "end")
813
+ self.lex_state = :expr_endarg
814
+
815
+ assert_lex3("x.y do 42 end",
816
+ nil,
817
+ :tIDENTIFIER, "x", :expr_end,
818
+ :tDOT, ".", :expr_dot,
819
+ :tIDENTIFIER, "y", :expr_arg,
820
+ :kDO_BLOCK, "do", :expr_beg,
821
+ :tINTEGER, 42, :expr_end,
822
+ :kEND, "end", :expr_end) do
823
+ @lex.cmdarg.push true
824
+ end
732
825
  end
733
826
 
734
827
  def test_yylex_do_block2
735
- @lex.lex_state = :expr_endarg
828
+ self.lex_state = :expr_endarg
736
829
 
737
- util_lex_token("do 42 end",
738
- :kDO_BLOCK, "do",
739
- :tINTEGER, 42,
740
- :kEND, "end")
830
+ assert_lex3("do 42 end",
831
+ nil,
832
+ :kDO_BLOCK, "do", :expr_beg,
833
+ :tINTEGER, 42, :expr_end,
834
+ :kEND, "end", :expr_end)
741
835
  end
742
836
 
743
837
  def test_yylex_do_cond
744
- @lex.cond.push true
745
-
746
- util_lex_token("x do 42 end",
747
- :tIDENTIFIER, "x",
748
- :kDO_COND, "do",
749
- :tINTEGER, 42,
750
- :kEND, "end")
838
+ assert_lex3("x do 42 end",
839
+ nil,
840
+ :tIDENTIFIER, "x", :expr_cmdarg,
841
+ :kDO_COND, "do", :expr_beg,
842
+ :tINTEGER, 42, :expr_end,
843
+ :kEND, "end", :expr_end) do
844
+ @lex.cond.push true
845
+ end
751
846
  end
752
847
 
753
848
  def test_yylex_dollar
754
- util_lex_token("$", "$", "$") # FIX: wtf is this?!?
849
+ assert_lex3("$", nil, "$", "$", :expr_end)
850
+ # FIX: wtf is this?!?
755
851
  end
756
852
 
757
853
  def test_yylex_dot # HINT message sends
758
- util_lex_token ".", :tDOT, "."
854
+ assert_lex3(".", nil, :tDOT, ".", :expr_dot)
759
855
  end
760
856
 
761
857
  def test_yylex_dot2
762
- util_lex_token "..", :tDOT2, ".."
858
+ assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
763
859
  end
764
860
 
765
861
  def test_yylex_dot3
766
- util_lex_token "...", :tDOT3, "..."
862
+ assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
767
863
  end
768
864
 
769
865
  def test_yylex_equals
770
- util_lex_token "=", :tEQL, "=" # FIX: this sucks
866
+ # FIX: this sucks
867
+ assert_lex3("=", nil, :tEQL, "=", :expr_beg)
771
868
  end
772
869
 
773
870
  def test_yylex_equals2
774
- util_lex_token "==", :tEQ, "=="
871
+ assert_lex3("==", nil, :tEQ, "==", :expr_beg)
775
872
  end
776
873
 
777
874
  def test_yylex_equals3
778
- util_lex_token "===", :tEQQ, "==="
875
+ assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
779
876
  end
780
877
 
781
878
  def test_yylex_equals_tilde
782
- util_lex_token "=~", :tMATCH, "=~"
879
+ assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
783
880
  end
784
881
 
785
882
  def test_yylex_float
786
- util_lex_token "1.0", :tFLOAT, 1.0
883
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
787
884
  end
788
885
 
789
886
  def test_yylex_float_bad_no_underscores
790
- util_bad_token "1__0.0"
887
+ refute_lex "1__0.0"
791
888
  end
792
889
 
793
890
  def test_yylex_float_bad_no_zero_leading
794
- util_bad_token ".0"
891
+ refute_lex ".0"
795
892
  end
796
893
 
797
894
  def test_yylex_float_bad_trailing_underscore
798
- util_bad_token "123_.0"
895
+ refute_lex "123_.0"
799
896
  end
800
897
 
801
898
  def test_yylex_float_call
802
- util_lex_token("1.0.to_s",
803
- :tFLOAT, 1.0,
804
- :tDOT, ".",
805
- :tIDENTIFIER, "to_s")
899
+ assert_lex3("1.0.to_s",
900
+ nil,
901
+ :tFLOAT, 1.0, :expr_end,
902
+ :tDOT, ".", :expr_dot,
903
+ :tIDENTIFIER, "to_s", :expr_arg)
806
904
  end
807
905
 
808
906
  def test_yylex_float_dot_E
809
- util_lex_token "1.0E10", :tFLOAT, 1.0e10
907
+ assert_lex3("1.0E10",
908
+ nil,
909
+ :tFLOAT, 10000000000.0, :expr_end)
810
910
  end
811
911
 
812
912
  def test_yylex_float_dot_E_neg
813
- util_lex_token("-1.0E10",
814
- :tUMINUS_NUM, "-",
815
- :tFLOAT, 1.0e10)
913
+ assert_lex3("-1.0E10",
914
+ nil,
915
+ :tUMINUS_NUM, "-", :expr_beg,
916
+ :tFLOAT, 10000000000.0, :expr_end)
816
917
  end
817
918
 
818
919
  def test_yylex_float_dot_e
819
- util_lex_token "1.0e10", :tFLOAT, 1.0e10
920
+ assert_lex3("1.0e10",
921
+ nil,
922
+ :tFLOAT, 10000000000.0, :expr_end)
820
923
  end
821
924
 
822
925
  def test_yylex_float_dot_e_neg
823
- util_lex_token("-1.0e10",
824
- :tUMINUS_NUM, "-",
825
- :tFLOAT, 1.0e10)
926
+ assert_lex3("-1.0e10",
927
+ nil,
928
+ :tUMINUS_NUM, "-", :expr_beg,
929
+ :tFLOAT, 10000000000.0, :expr_end)
826
930
  end
827
931
 
828
932
  def test_yylex_float_e
829
- util_lex_token "1e10", :tFLOAT, 1e10
933
+ assert_lex3("1e10",
934
+ nil,
935
+ :tFLOAT, 10000000000.0, :expr_end)
830
936
  end
831
937
 
832
938
  def test_yylex_float_e_bad_double_e
833
- util_bad_token "1e2e3"
939
+ refute_lex "1e2e3"
834
940
  end
835
941
 
836
942
  def test_yylex_float_e_bad_trailing_underscore
837
- util_bad_token "123_e10"
943
+ refute_lex "123_e10"
838
944
  end
839
945
 
840
946
  def test_yylex_float_e_minus
841
- util_lex_token "1e-10", :tFLOAT, 1e-10
947
+ assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
842
948
  end
843
949
 
844
950
  def test_yylex_float_e_neg
845
- util_lex_token("-1e10",
846
- :tUMINUS_NUM, "-",
847
- :tFLOAT, 1e10)
951
+ assert_lex3("-1e10",
952
+ nil,
953
+ :tUMINUS_NUM, "-", :expr_beg,
954
+ :tFLOAT, 10000000000.0, :expr_end)
848
955
  end
849
956
 
850
957
  def test_yylex_float_e_neg_minus
851
- util_lex_token("-1e-10",
852
- :tUMINUS_NUM, "-",
853
- :tFLOAT, 1e-10)
958
+ assert_lex3("-1e-10",
959
+ nil,
960
+ :tUMINUS_NUM, "-", :expr_beg,
961
+ :tFLOAT, 1.0e-10, :expr_end)
854
962
  end
855
963
 
856
964
  def test_yylex_float_e_neg_plus
857
- util_lex_token("-1e+10",
858
- :tUMINUS_NUM, "-",
859
- :tFLOAT, 1e10)
965
+ assert_lex3("-1e+10",
966
+ nil,
967
+ :tUMINUS_NUM, "-", :expr_beg,
968
+ :tFLOAT, 10000000000.0, :expr_end)
860
969
  end
861
970
 
862
971
  def test_yylex_float_e_plus
863
- util_lex_token "1e+10", :tFLOAT, 1e10
972
+ assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
864
973
  end
865
974
 
866
975
  def test_yylex_float_e_zero
867
- util_lex_token "0e0", :tFLOAT, 0e0
976
+ assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
868
977
  end
869
978
 
870
979
  def test_yylex_float_neg
871
- util_lex_token("-1.0",
872
- :tUMINUS_NUM, "-",
873
- :tFLOAT, 1.0)
980
+ assert_lex3("-1.0",
981
+ nil,
982
+ :tUMINUS_NUM, "-", :expr_beg,
983
+ :tFLOAT, 1.0, :expr_end)
874
984
  end
875
985
 
876
986
  def test_yylex_ge
877
- util_lex_token("a >= 2",
878
- :tIDENTIFIER, "a",
879
- :tGEQ, ">=",
880
- :tINTEGER, 2)
987
+ assert_lex3("a >= 2",
988
+ nil,
989
+ :tIDENTIFIER, "a", :expr_cmdarg,
990
+ :tGEQ, ">=", :expr_beg,
991
+ :tINTEGER, 2, :expr_end)
881
992
  end
882
993
 
883
994
  def test_yylex_global
884
- util_lex_token("$blah", :tGVAR, "$blah")
995
+ assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
885
996
  end
886
997
 
887
998
  def test_yylex_global_backref
888
- @lex.lex_state = :expr_fname
889
- util_lex_token("$`", :tGVAR, "$`")
999
+ self.lex_state = :expr_fname
1000
+
1001
+ assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
890
1002
  end
891
1003
 
892
1004
  def test_yylex_global_dash_nothing
893
- util_lex_token("$- ", :tGVAR, "$-")
1005
+ assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
894
1006
  end
895
1007
 
896
1008
  def test_yylex_global_dash_something
897
- util_lex_token("$-x", :tGVAR, "$-x")
1009
+ assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
898
1010
  end
899
1011
 
900
1012
  def test_yylex_global_number
901
- @lex.lex_state = :expr_fname
902
- util_lex_token("$1", :tGVAR, "$1")
1013
+ self.lex_state = :expr_fname
1014
+
1015
+ assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
903
1016
  end
904
1017
 
905
1018
  def test_yylex_global_number_big
906
- @lex.lex_state = :expr_fname
907
- util_lex_token("$1234", :tGVAR, "$1234")
1019
+ self.lex_state = :expr_fname
1020
+
1021
+ assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
908
1022
  end
909
1023
 
910
1024
  def test_yylex_global_other
911
- util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
912
- :tLBRACK, "[",
913
- :tGVAR, "$~", :tCOMMA, ",",
914
- :tGVAR, "$*", :tCOMMA, ",",
915
- :tGVAR, "$$", :tCOMMA, ",",
916
- :tGVAR, "$\?", :tCOMMA, ",",
917
- :tGVAR, "$!", :tCOMMA, ",",
918
- :tGVAR, "$@", :tCOMMA, ",",
919
- :tGVAR, "$/", :tCOMMA, ",",
920
- :tGVAR, "$\\", :tCOMMA, ",",
921
- :tGVAR, "$;", :tCOMMA, ",",
922
- :tGVAR, "$,", :tCOMMA, ",",
923
- :tGVAR, "$.", :tCOMMA, ",",
924
- :tGVAR, "$=", :tCOMMA, ",",
925
- :tGVAR, "$:", :tCOMMA, ",",
926
- :tGVAR, "$<", :tCOMMA, ",",
927
- :tGVAR, "$>", :tCOMMA, ",",
928
- :tGVAR, "$\"",
929
- :tRBRACK, "]")
1025
+ assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
1026
+ nil,
1027
+ :tLBRACK, "[", :expr_beg,
1028
+ :tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
1029
+ :tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
1030
+ :tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
1031
+ :tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
1032
+ :tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
1033
+ :tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
1034
+ :tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
1035
+ :tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
1036
+ :tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
1037
+ :tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
1038
+ :tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
1039
+ :tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
1040
+ :tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
1041
+ :tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
1042
+ :tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
1043
+ :tGVAR, "$\"", :expr_end,
1044
+ :tRBRACK, "]", :expr_endarg)
930
1045
  end
931
1046
 
932
1047
  def test_yylex_global_underscore
933
- util_lex_token("$_",
934
- :tGVAR, "$_")
1048
+ assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
935
1049
  end
936
1050
 
937
1051
  def test_yylex_global_wierd
938
- util_lex_token("$__blah",
939
- :tGVAR, "$__blah")
1052
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
940
1053
  end
941
1054
 
942
1055
  def test_yylex_global_zero
943
- util_lex_token("$0", :tGVAR, "$0")
1056
+ assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
944
1057
  end
945
1058
 
946
1059
  def test_yylex_gt
947
- util_lex_token("a > 2",
948
- :tIDENTIFIER, "a",
949
- :tGT, ">",
950
- :tINTEGER, 2)
1060
+ assert_lex3("a > 2",
1061
+ nil,
1062
+ :tIDENTIFIER, "a", :expr_cmdarg,
1063
+ :tGT, ">", :expr_beg,
1064
+ :tINTEGER, 2, :expr_end)
951
1065
  end
952
1066
 
953
1067
  def test_yylex_heredoc_backtick
954
- util_lex_token("a = <<`EOF`\n blah blah\nEOF\n",
955
- :tIDENTIFIER, "a",
956
- :tEQL, "=",
957
- :tXSTRING_BEG, "`",
958
- :tSTRING_CONTENT, " blah blah\n",
959
- :tSTRING_END, "EOF",
960
- :tNL, nil)
1068
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1069
+ nil,
1070
+ :tIDENTIFIER, "a", :expr_cmdarg,
1071
+ :tEQL, "=", :expr_beg,
1072
+ :tXSTRING_BEG, "`", :expr_beg,
1073
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1074
+ :tSTRING_END, "EOF", :expr_end,
1075
+ :tNL, nil, :expr_beg)
961
1076
  end
962
1077
 
963
1078
  def test_yylex_heredoc_double
964
- util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n",
965
- :tIDENTIFIER, "a",
966
- :tEQL, "=",
967
- :tSTRING_BEG, "\"",
968
- :tSTRING_CONTENT, " blah blah\n",
969
- :tSTRING_END, "EOF",
970
- :tNL, nil)
1079
+ assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1080
+ nil,
1081
+ :tIDENTIFIER, "a", :expr_cmdarg,
1082
+ :tEQL, "=", :expr_beg,
1083
+ :tSTRING_BEG, "\"", :expr_beg,
1084
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1085
+ :tSTRING_END, "EOF", :expr_end,
1086
+ :tNL, nil, :expr_beg)
971
1087
  end
972
1088
 
973
1089
  def test_yylex_heredoc_double_dash
974
- util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n",
975
- :tIDENTIFIER, "a",
976
- :tEQL, "=",
977
- :tSTRING_BEG, "\"",
978
- :tSTRING_CONTENT, " blah blah\n",
979
- :tSTRING_END, "EOF",
980
- :tNL, nil)
1090
+ assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1091
+ nil,
1092
+ :tIDENTIFIER, "a", :expr_cmdarg,
1093
+ :tEQL, "=", :expr_beg,
1094
+ :tSTRING_BEG, "\"", :expr_beg,
1095
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1096
+ :tSTRING_END, "EOF", :expr_end,
1097
+ :tNL, nil, :expr_beg)
981
1098
  end
982
1099
 
983
1100
  def test_yylex_heredoc_double_eos
984
- util_bad_token("a = <<\"EOF\"\nblah",
985
- :tIDENTIFIER, "a",
986
- :tEQL, "=",
987
- :tSTRING_BEG, "\"")
1101
+ refute_lex("a = <<\"EOF\"\nblah",
1102
+ :tIDENTIFIER, "a",
1103
+ :tEQL, "=",
1104
+ :tSTRING_BEG, "\"")
988
1105
  end
989
1106
 
990
1107
  def test_yylex_heredoc_double_eos_nl
991
- util_bad_token("a = <<\"EOF\"\nblah\n",
992
- :tIDENTIFIER, "a",
993
- :tEQL, "=",
994
- :tSTRING_BEG, "\"")
1108
+ refute_lex("a = <<\"EOF\"\nblah\n",
1109
+ :tIDENTIFIER, "a",
1110
+ :tEQL, "=",
1111
+ :tSTRING_BEG, "\"")
995
1112
  end
996
1113
 
997
1114
  def test_yylex_heredoc_double_interp
998
- # TODO: convert to assert_lex
999
- util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
1000
- :tIDENTIFIER, "a",
1001
- :tEQL, "=",
1002
- :tSTRING_BEG, "\"",
1003
- :tSTRING_CONTENT, "#x a ",
1004
- :tSTRING_DVAR, "\#@",
1005
- :tSTRING_CONTENT, "@a b ", # HUH?
1006
- :tSTRING_DVAR, "\#$",
1007
- :tSTRING_CONTENT, "$b c ", # HUH?
1008
- :tSTRING_DBEG, "\#{",
1009
- :tSTRING_CONTENT, "3} \n", # HUH?
1010
- :tSTRING_END, "EOF",
1011
- :tNL, nil)
1115
+ assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
1116
+ nil,
1117
+ :tIDENTIFIER, "a", :expr_cmdarg,
1118
+ :tEQL, "=", :expr_beg,
1119
+ :tSTRING_BEG, "\"", :expr_beg,
1120
+ :tSTRING_CONTENT, "#x a ", :expr_beg,
1121
+ :tSTRING_DVAR, "\#@", :expr_beg,
1122
+ :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
1123
+ :tSTRING_DVAR, "\#$", :expr_beg,
1124
+ :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
1125
+ :tSTRING_DBEG, "\#{", :expr_beg,
1126
+ :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
1127
+ :tSTRING_END, "EOF", :expr_end,
1128
+ :tNL, nil, :expr_beg)
1012
1129
  end
1013
1130
 
1014
1131
  def test_yylex_heredoc_empty
1015
- util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
1016
- :tSTRING_BEG, "\"",
1017
- :tSTRING_DBEG, "\#{",
1018
- :tSTRING_CONTENT, "x}\nblah2\n",
1019
- :tSTRING_END, "",
1020
- :tNL, nil)
1132
+ assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1133
+ nil,
1134
+ :tSTRING_BEG, "\"", :expr_beg,
1135
+ :tSTRING_DBEG, "\#{", :expr_beg,
1136
+ :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
1137
+ :tSTRING_END, "", :expr_end,
1138
+ :tNL, nil, :expr_beg)
1021
1139
  end
1022
1140
 
1023
1141
  def test_yylex_heredoc_none
1024
- util_lex_token("a = <<EOF\nblah\nblah\nEOF",
1025
- :tIDENTIFIER, "a",
1026
- :tEQL, "=",
1027
- :tSTRING_BEG, "\"",
1028
- :tSTRING_CONTENT, "blah\nblah\n",
1029
- :tSTRING_END, "EOF",
1030
- :tNL, nil)
1142
+ assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
1143
+ nil,
1144
+ :tIDENTIFIER, "a", :expr_cmdarg,
1145
+ :tEQL, "=", :expr_beg,
1146
+ :tSTRING_BEG, "\"", :expr_beg,
1147
+ :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1148
+ :tSTRING_END, "EOF", :expr_end,
1149
+ :tNL, nil, :expr_beg)
1031
1150
  end
1032
1151
 
1033
1152
  def test_yylex_heredoc_none_bad_eos
1034
- util_bad_token("a = <<EOF",
1035
- :tIDENTIFIER, "a",
1036
- :tEQL, "=",
1037
- :tSTRING_BEG, "\"")
1153
+ refute_lex("a = <<EOF",
1154
+ :tIDENTIFIER, "a",
1155
+ :tEQL, "=",
1156
+ :tSTRING_BEG, "\"")
1038
1157
  end
1039
1158
 
1040
1159
  def test_yylex_heredoc_none_dash
1041
- util_lex_token("a = <<-EOF\nblah\nblah\n EOF",
1042
- :tIDENTIFIER, "a",
1043
- :tEQL, "=",
1044
- :tSTRING_BEG, "\"",
1045
- :tSTRING_CONTENT, "blah\nblah\n",
1046
- :tSTRING_END, "EOF",
1047
- :tNL, nil)
1160
+ assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
1161
+ nil,
1162
+ :tIDENTIFIER, "a", :expr_cmdarg,
1163
+ :tEQL, "=", :expr_beg,
1164
+ :tSTRING_BEG, "\"", :expr_beg,
1165
+ :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1166
+ :tSTRING_END, "EOF", :expr_end,
1167
+ :tNL, nil, :expr_beg)
1048
1168
  end
1049
1169
 
1050
1170
  def test_yylex_heredoc_single
1051
- util_lex_token("a = <<'EOF'\n blah blah\nEOF\n",
1052
- :tIDENTIFIER, "a",
1053
- :tEQL, "=",
1054
- :tSTRING_BEG, "\"",
1055
- :tSTRING_CONTENT, " blah blah\n",
1056
- :tSTRING_END, "EOF",
1057
- :tNL, nil)
1171
+ assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1172
+ nil,
1173
+ :tIDENTIFIER, "a", :expr_cmdarg,
1174
+ :tEQL, "=", :expr_beg,
1175
+ :tSTRING_BEG, "\"", :expr_beg,
1176
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1177
+ :tSTRING_END, "EOF", :expr_end,
1178
+ :tNL, nil, :expr_beg)
1058
1179
  end
1059
1180
 
1060
1181
  def test_yylex_heredoc_single_bad_eos_body
1061
- util_bad_token("a = <<'EOF'\nblah",
1062
- :tIDENTIFIER, "a",
1063
- :tEQL, "=",
1064
- :tSTRING_BEG, "\"")
1182
+ refute_lex("a = <<'EOF'\nblah",
1183
+ :tIDENTIFIER, "a",
1184
+ :tEQL, "=",
1185
+ :tSTRING_BEG, "\"")
1065
1186
  end
1066
1187
 
1067
1188
  def test_yylex_heredoc_single_bad_eos_empty
1068
- util_bad_token("a = <<''\n",
1069
- :tIDENTIFIER, "a",
1070
- :tEQL, "=",
1071
- :tSTRING_BEG, "\"")
1189
+ refute_lex("a = <<''\n",
1190
+ :tIDENTIFIER, "a",
1191
+ :tEQL, "=",
1192
+ :tSTRING_BEG, "\"")
1072
1193
  end
1073
1194
 
1074
1195
  def test_yylex_heredoc_single_bad_eos_term
1075
- util_bad_token("a = <<'EOF",
1076
- :tIDENTIFIER, "a",
1077
- :tEQL, "=",
1078
- :tSTRING_BEG, "\"")
1196
+ refute_lex("a = <<'EOF",
1197
+ :tIDENTIFIER, "a",
1198
+ :tEQL, "=",
1199
+ :tSTRING_BEG, "\"")
1079
1200
  end
1080
1201
 
1081
1202
  def test_yylex_heredoc_single_bad_eos_term_nl
1082
- util_bad_token("a = <<'EOF\ns = 'blah blah'",
1083
- :tIDENTIFIER, "a",
1084
- :tEQL, "=",
1085
- :tSTRING_BEG, "\"")
1203
+ refute_lex("a = <<'EOF\ns = 'blah blah'",
1204
+ :tIDENTIFIER, "a",
1205
+ :tEQL, "=",
1206
+ :tSTRING_BEG, "\"")
1086
1207
  end
1087
1208
 
1088
1209
  def test_yylex_heredoc_single_dash
1089
- util_lex_token("a = <<-'EOF'\n blah blah\n EOF\n",
1090
- :tIDENTIFIER, "a",
1091
- :tEQL, "=",
1092
- :tSTRING_BEG, "\"",
1093
- :tSTRING_CONTENT, " blah blah\n",
1094
- :tSTRING_END, "EOF",
1095
- :tNL, nil)
1210
+ assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
1211
+ nil,
1212
+ :tIDENTIFIER, "a", :expr_cmdarg,
1213
+ :tEQL, "=", :expr_beg,
1214
+ :tSTRING_BEG, "\"", :expr_beg,
1215
+ :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1216
+ :tSTRING_END, "EOF", :expr_end,
1217
+ :tNL, nil, :expr_beg)
1096
1218
  end
1097
1219
 
1098
1220
  def test_yylex_identifier
1099
- util_lex_token("identifier", :tIDENTIFIER, "identifier")
1221
+ assert_lex3("identifier",
1222
+ nil,
1223
+ :tIDENTIFIER, "identifier", :expr_cmdarg)
1100
1224
  end
1101
1225
 
1102
1226
  def test_yylex_identifier_bang
1103
- util_lex_token("identifier!", :tFID, "identifier!")
1227
+ assert_lex3("identifier!",
1228
+ nil,
1229
+ :tFID, "identifier!", :expr_cmdarg)
1104
1230
  end
1105
1231
 
1106
1232
  def test_yylex_identifier_cmp
1107
- util_lex_fname "<=>", :tCMP
1233
+ assert_lex_fname "<=>", :tCMP
1108
1234
  end
1109
1235
 
1110
1236
  def test_yylex_identifier_def__18
1111
1237
  setup_lexer_class Ruby18Parser
1112
1238
 
1113
- util_lex_fname "identifier", :tIDENTIFIER, :expr_end
1239
+ assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
1114
1240
  end
1115
1241
 
1116
1242
  def test_yylex_identifier_def__1920
1117
1243
  setup_lexer_class Ruby19Parser
1118
1244
 
1119
- util_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1245
+ assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1120
1246
  end
1121
1247
 
1122
1248
  def test_yylex_identifier_eh
1123
- util_lex_token("identifier?", :tFID, "identifier?")
1249
+ assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
1124
1250
  end
1125
1251
 
1126
1252
  def test_yylex_identifier_equals_arrow
1127
- @lex.lex_state = :expr_fname
1128
- util_lex_token(":blah==>",
1129
- :tSYMBOL, "blah=",
1130
- :tASSOC, "=>")
1253
+ assert_lex3(":blah==>",
1254
+ nil,
1255
+ :tSYMBOL, "blah=", :expr_end,
1256
+ :tASSOC, "=>", :expr_beg)
1131
1257
  end
1132
1258
 
1133
1259
  def test_yylex_identifier_equals3
1134
- # @lex.lex_state = :expr_fname
1135
- util_lex_token(":a===b",
1136
- :tSYMBOL, "a",
1137
- :tEQQ, "===",
1138
- :tIDENTIFIER, "b")
1260
+ assert_lex3(":a===b",
1261
+ nil,
1262
+ :tSYMBOL, "a", :expr_end,
1263
+ :tEQQ, "===", :expr_beg,
1264
+ :tIDENTIFIER, "b", :expr_arg)
1139
1265
  end
1140
1266
 
1141
1267
  def test_yylex_identifier_equals_equals_arrow
1142
- # @lex.lex_state = :expr_fname
1143
- util_lex_token(":a==>b",
1144
- :tSYMBOL, "a=",
1145
- :tASSOC, "=>",
1146
- :tIDENTIFIER, "b")
1268
+ assert_lex3(":a==>b",
1269
+ nil,
1270
+ :tSYMBOL, "a=", :expr_end,
1271
+ :tASSOC, "=>", :expr_beg,
1272
+ :tIDENTIFIER, "b", :expr_arg)
1147
1273
  end
1148
1274
 
1149
1275
  def test_yylex_identifier_equals_caret
1150
- util_lex_fname "^", :tCARET
1276
+ assert_lex_fname "^", :tCARET
1151
1277
  end
1152
1278
 
1153
1279
  def test_yylex_identifier_equals_def__18
1154
1280
  setup_lexer_class Ruby18Parser
1155
1281
 
1156
- util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
1282
+ assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
1157
1283
  end
1158
1284
 
1159
1285
  def test_yylex_identifier_equals_def__1920
1160
1286
  setup_lexer_class Ruby19Parser
1161
1287
 
1162
- util_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1288
+ assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1163
1289
  end
1164
1290
 
1165
1291
  def test_yylex_identifier_equals_def2
1166
- util_lex_fname "==", :tEQ
1292
+ assert_lex_fname "==", :tEQ
1167
1293
  end
1168
1294
 
1169
1295
  def test_yylex_identifier_equals_expr
1170
- @lex.lex_state = :expr_dot
1171
- util_lex_token("y = arg",
1172
- :tIDENTIFIER, "y",
1173
- :tEQL, "=",
1174
- :tIDENTIFIER, "arg")
1175
-
1176
- assert_equal :expr_arg, @lex.lex_state
1296
+ self.lex_state = :expr_dot
1297
+ assert_lex3("y = arg",
1298
+ nil,
1299
+ :tIDENTIFIER, "y", :expr_cmdarg,
1300
+ :tEQL, "=", :expr_beg,
1301
+ :tIDENTIFIER, "arg", :expr_arg)
1177
1302
  end
1178
1303
 
1179
1304
  def test_yylex_identifier_equals_or
1180
- util_lex_fname "|", :tPIPE
1305
+ assert_lex_fname "|", :tPIPE
1181
1306
  end
1182
1307
 
1183
1308
  def test_yylex_identifier_equals_slash
1184
- util_lex_fname "/", :tDIVIDE
1309
+ assert_lex_fname "/", :tDIVIDE
1185
1310
  end
1186
1311
 
1187
1312
  def test_yylex_identifier_equals_tilde
1188
- @lex.lex_state = :expr_fname # can only set via parser's defs
1189
- util_lex_token("identifier=~",
1190
- :tIDENTIFIER, "identifier",
1191
- :tMATCH, "=~")
1313
+ self.lex_state = :expr_fname # can only set via parser's defs
1314
+
1315
+ assert_lex3("identifier=~",
1316
+ nil,
1317
+ :tIDENTIFIER, "identifier", :expr_endfn,
1318
+ :tMATCH, "=~", :expr_beg)
1192
1319
  end
1193
1320
 
1194
1321
  def test_yylex_identifier_gt
1195
- util_lex_fname ">", :tGT
1322
+ assert_lex_fname ">", :tGT
1196
1323
  end
1197
1324
 
1198
1325
  def test_yylex_identifier_le
1199
- util_lex_fname "<=", :tLEQ
1326
+ assert_lex_fname "<=", :tLEQ
1200
1327
  end
1201
1328
 
1202
1329
  def test_yylex_identifier_lt
1203
- util_lex_fname "<", :tLT
1330
+ assert_lex_fname "<", :tLT
1204
1331
  end
1205
1332
 
1206
1333
  def test_yylex_identifier_tilde
1207
- util_lex_fname "~", :tTILDE
1334
+ assert_lex_fname "~", :tTILDE
1208
1335
  end
1209
1336
 
1210
1337
  def test_yylex_index
1211
- util_lex_fname "[]", :tAREF
1338
+ assert_lex_fname "[]", :tAREF
1212
1339
  end
1213
1340
 
1214
1341
  def test_yylex_index_equals
1215
- util_lex_fname "[]=", :tASET
1342
+ assert_lex_fname "[]=", :tASET
1216
1343
  end
1217
1344
 
1218
1345
  def test_yylex_integer
1219
- util_lex_token "42", :tINTEGER, 42
1346
+ assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1220
1347
  end
1221
1348
 
1222
1349
  def test_yylex_integer_bin
1223
- util_lex_token "0b101010", :tINTEGER, 42
1350
+ assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
1224
1351
  end
1225
1352
 
1226
1353
  def test_yylex_integer_bin_bad_none
1227
- util_bad_token "0b "
1354
+ refute_lex "0b "
1228
1355
  end
1229
1356
 
1230
1357
  def test_yylex_integer_bin_bad_underscores
1231
- util_bad_token "0b10__01"
1358
+ refute_lex "0b10__01"
1232
1359
  end
1233
1360
 
1234
1361
  def test_yylex_integer_dec
1235
- util_lex_token "42", :tINTEGER, 42
1362
+ assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1236
1363
  end
1237
1364
 
1238
1365
  def test_yylex_integer_dec_bad_underscores
1239
- util_bad_token "42__24"
1366
+ refute_lex "42__24"
1240
1367
  end
1241
1368
 
1242
1369
  def test_yylex_integer_dec_d
1243
- util_lex_token "0d42", :tINTEGER, 42
1370
+ assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
1244
1371
  end
1245
1372
 
1246
1373
  def test_yylex_integer_dec_d_bad_none
1247
- util_bad_token "0d"
1374
+ refute_lex "0d"
1248
1375
  end
1249
1376
 
1250
1377
  def test_yylex_integer_dec_d_bad_underscores
1251
- util_bad_token "0d42__24"
1378
+ refute_lex "0d42__24"
1252
1379
  end
1253
1380
 
1254
1381
  def test_yylex_question_eh_a__18
1255
1382
  setup_lexer_class Ruby18Parser
1256
1383
 
1257
- util_lex_token "?a", :tINTEGER, 97
1384
+ assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
1258
1385
  end
1259
1386
 
1260
1387
  def test_yylex_question_eh_a__19
1261
1388
  setup_lexer_class Ruby19Parser
1262
1389
 
1263
- util_lex_token '?a', :tSTRING, "a"
1390
+ assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
1264
1391
  end
1265
1392
 
1266
1393
  def test_yylex_question_eh_escape_M_escape_C__18
1267
1394
  setup_lexer_class Ruby18Parser
1268
1395
 
1269
- util_lex_token '?\M-\C-a', :tINTEGER, 129
1396
+ assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
1270
1397
  end
1271
1398
 
1272
1399
  def test_yylex_question_eh_escape_M_escape_C__19
1273
1400
  setup_lexer_class Ruby19Parser
1274
1401
 
1275
- util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
1402
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
1276
1403
  end
1277
1404
 
1278
1405
  def test_yylex_integer_hex
1279
- util_lex_token "0x2a", :tINTEGER, 42
1406
+ assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
1280
1407
  end
1281
1408
 
1282
1409
  def test_yylex_integer_hex_bad_none
1283
- util_bad_token "0x "
1410
+ refute_lex "0x "
1284
1411
  end
1285
1412
 
1286
1413
  def test_yylex_integer_hex_bad_underscores
1287
- util_bad_token "0xab__cd"
1414
+ refute_lex "0xab__cd"
1288
1415
  end
1289
1416
 
1290
1417
  def test_yylex_integer_oct
1291
- util_lex_token "052", :tINTEGER, 42
1418
+ assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
1292
1419
  end
1293
1420
 
1294
1421
  def test_yylex_integer_oct_bad_range
1295
- util_bad_token "08"
1422
+ refute_lex "08"
1296
1423
  end
1297
1424
 
1298
1425
  def test_yylex_integer_oct_bad_range2
1299
- util_bad_token "08"
1426
+ refute_lex "08"
1300
1427
  end
1301
1428
 
1302
1429
  def test_yylex_integer_oct_bad_underscores
1303
- util_bad_token "01__23"
1430
+ refute_lex "01__23"
1304
1431
  end
1305
1432
 
1306
1433
  def test_yylex_integer_oct_O
1307
- util_lex_token "0O52", :tINTEGER, 42
1434
+ assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
1308
1435
  end
1309
1436
 
1310
1437
  def test_yylex_integer_oct_O_bad_range
1311
- util_bad_token "0O8"
1438
+ refute_lex "0O8"
1312
1439
  end
1313
1440
 
1314
1441
  def test_yylex_integer_oct_O_bad_underscores
1315
- util_bad_token "0O1__23"
1442
+ refute_lex "0O1__23"
1316
1443
  end
1317
1444
 
1318
1445
  def test_yylex_integer_oct_O_not_bad_none
1319
- util_lex_token "0O ", :tINTEGER, 0
1446
+ assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
1320
1447
  end
1321
1448
 
1322
1449
  def test_yylex_integer_oct_o
1323
- util_lex_token "0o52", :tINTEGER, 42
1450
+ assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
1324
1451
  end
1325
1452
 
1326
1453
  def test_yylex_integer_oct_o_bad_range
1327
- util_bad_token "0o8"
1454
+ refute_lex "0o8"
1328
1455
  end
1329
1456
 
1330
1457
  def test_yylex_integer_oct_o_bad_underscores
1331
- util_bad_token "0o1__23"
1458
+ refute_lex "0o1__23"
1332
1459
  end
1333
1460
 
1334
1461
  def test_yylex_integer_oct_o_not_bad_none
1335
- util_lex_token "0o ", :tINTEGER, 0
1462
+ assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
1336
1463
  end
1337
1464
 
1338
1465
  def test_yylex_integer_trailing
1339
- util_lex_token("1.to_s",
1340
- :tINTEGER, 1,
1341
- :tDOT, '.',
1342
- :tIDENTIFIER, 'to_s')
1466
+ assert_lex3("1.to_s",
1467
+ nil,
1468
+ :tINTEGER, 1, :expr_end,
1469
+ :tDOT, ".", :expr_dot,
1470
+ :tIDENTIFIER, "to_s", :expr_arg)
1343
1471
  end
1344
1472
 
1345
1473
  def test_yylex_integer_underscore
1346
- util_lex_token "4_2", :tINTEGER, 42
1474
+ assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
1347
1475
  end
1348
1476
 
1349
1477
  def test_yylex_integer_underscore_bad
1350
- util_bad_token "4__2"
1478
+ refute_lex "4__2"
1351
1479
  end
1352
1480
 
1353
1481
  def test_yylex_integer_zero
1354
- util_lex_token "0", :tINTEGER, 0
1482
+ assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
1355
1483
  end
1356
1484
 
1357
1485
  def test_yylex_ivar
1358
- util_lex_token "@blah", :tIVAR, "@blah"
1486
+ assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
1359
1487
  end
1360
1488
 
1361
1489
  def test_yylex_ivar_bad
1362
- util_bad_token "@1"
1490
+ refute_lex "@1"
1363
1491
  end
1364
1492
 
1365
1493
  def test_yylex_ivar_bad_0_length
1366
- util_bad_token "1+@\n", :tINTEGER, 1, :tPLUS, "+"
1494
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
1367
1495
  end
1368
1496
 
1369
1497
  def test_yylex_keyword_expr
1370
- @lex.lex_state = :expr_endarg
1371
-
1372
- util_lex_token("if", :kIF_MOD, "if")
1498
+ self.lex_state = :expr_endarg
1373
1499
 
1374
- assert_equal :expr_beg, @lex.lex_state
1500
+ assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
1375
1501
  end
1376
1502
 
1377
1503
  def test_yylex_lt
1378
- util_lex_token "<", :tLT, "<"
1504
+ assert_lex3("<", nil, :tLT, "<", :expr_beg)
1379
1505
  end
1380
1506
 
1381
1507
  def test_yylex_lt2
1382
- util_lex_token("a <\< b",
1383
- :tIDENTIFIER, "a",
1384
- :tLSHFT, "<\<",
1385
- :tIDENTIFIER, "b")
1386
-
1508
+ assert_lex3("a << b",
1509
+ nil,
1510
+ :tIDENTIFIER, "a", :expr_cmdarg,
1511
+ :tLSHFT, "<<", :expr_beg,
1512
+ :tIDENTIFIER, "b", :expr_arg)
1387
1513
  end
1388
1514
 
1389
1515
  def test_yylex_lt2_equals
1390
- util_lex_token("a <\<= b",
1391
- :tIDENTIFIER, "a",
1392
- :tOP_ASGN, "<\<",
1393
- :tIDENTIFIER, "b")
1516
+ assert_lex3("a <<= b",
1517
+ nil,
1518
+ :tIDENTIFIER, "a", :expr_cmdarg,
1519
+ :tOP_ASGN, "<<", :expr_beg,
1520
+ :tIDENTIFIER, "b", :expr_arg)
1394
1521
  end
1395
1522
 
1396
1523
  def test_yylex_lt_equals
1397
- util_lex_token "<=", :tLEQ, "<="
1524
+ assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
1398
1525
  end
1399
1526
 
1400
1527
  def test_yylex_minus
1401
- util_lex_token("1 - 2",
1402
- :tINTEGER, 1,
1403
- :tMINUS, "-",
1404
- :tINTEGER, 2)
1528
+ assert_lex3("1 - 2",
1529
+ nil,
1530
+ :tINTEGER, 1, :expr_end,
1531
+ :tMINUS, "-", :expr_beg,
1532
+ :tINTEGER, 2, :expr_end)
1405
1533
  end
1406
1534
 
1407
1535
  def test_yylex_minus_equals
1408
- util_lex_token "-=", :tOP_ASGN, "-"
1536
+ assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
1409
1537
  end
1410
1538
 
1411
1539
  def test_yylex_minus_method
1412
- @lex.lex_state = :expr_fname
1413
- util_lex_token "-", :tMINUS, "-"
1540
+ self.lex_state = :expr_fname
1541
+
1542
+ assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
1414
1543
  end
1415
1544
 
1416
1545
  def test_yylex_minus_unary_method
1417
- @lex.lex_state = :expr_fname
1418
- util_lex_token "-@", :tUMINUS, "-@"
1546
+ self.lex_state = :expr_fname
1547
+
1548
+ assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
1419
1549
  end
1420
1550
 
1421
1551
  def test_yylex_minus_unary_number
1422
- util_lex_token("-42",
1423
- :tUMINUS_NUM, "-",
1424
- :tINTEGER, 42)
1552
+ assert_lex3("-42",
1553
+ nil,
1554
+ :tUMINUS_NUM, "-", :expr_beg,
1555
+ :tINTEGER, 42, :expr_end)
1425
1556
  end
1426
1557
 
1427
1558
  def test_yylex_nth_ref
1428
- util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
1429
- :tLBRACK, "[",
1430
- :tNTH_REF, 1, :tCOMMA, ",",
1431
- :tNTH_REF, 2, :tCOMMA, ",",
1432
- :tNTH_REF, 3, :tCOMMA, ",",
1433
- :tNTH_REF, 4, :tCOMMA, ",",
1434
- :tNTH_REF, 5, :tCOMMA, ",",
1435
- :tNTH_REF, 6, :tCOMMA, ",",
1436
- :tNTH_REF, 7, :tCOMMA, ",",
1437
- :tNTH_REF, 8, :tCOMMA, ",",
1438
- :tNTH_REF, 9,
1439
- :tRBRACK, "]")
1559
+ assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
1560
+ nil,
1561
+ :tLBRACK, "[", :expr_beg,
1562
+ :tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1563
+ :tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1564
+ :tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
1565
+ :tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
1566
+ :tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
1567
+ :tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
1568
+ :tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
1569
+ :tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
1570
+ :tNTH_REF, 9, :expr_end,
1571
+ :tRBRACK, "]", :expr_endarg)
1440
1572
  end
1441
1573
 
1442
1574
  def test_yylex_open_bracket
1443
- util_lex_token("(", :tLPAREN, "(")
1575
+ assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
1444
1576
  end
1445
1577
 
1446
1578
  def test_yylex_open_bracket_cmdarg
1447
- @lex.lex_state = :expr_cmdarg
1448
- util_lex_token(" (", :tLPAREN_ARG, "(")
1579
+ self.lex_state = :expr_cmdarg
1580
+
1581
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1449
1582
  end
1450
1583
 
1451
1584
  def test_yylex_open_bracket_exprarg__18
1452
1585
  setup_lexer_class Ruby18Parser
1586
+ self.lex_state = :expr_arg
1453
1587
 
1454
- @lex.lex_state = :expr_arg
1455
- util_lex_token(" (", :tLPAREN2, "(")
1588
+ assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
1456
1589
  end
1457
1590
 
1458
1591
  def test_yylex_open_bracket_exprarg__19
1459
1592
  setup_lexer_class Ruby19Parser
1593
+ self.lex_state = :expr_arg
1460
1594
 
1461
- @lex.lex_state = :expr_arg
1462
- util_lex_token(" (", :tLPAREN_ARG, "(")
1595
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1463
1596
  end
1464
1597
 
1465
1598
  def test_yylex_open_curly_bracket
1466
- util_lex_token("{",
1467
- :tLBRACE, "{")
1599
+ assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
1468
1600
  end
1469
1601
 
1470
1602
  def test_yylex_open_curly_bracket_arg
1471
- @lex.lex_state = :expr_arg
1472
- util_lex_token("m { 3 }",
1473
- :tIDENTIFIER, "m",
1474
- :tLCURLY, "{",
1475
- :tINTEGER, 3,
1476
- :tRCURLY, "}")
1603
+ self.lex_state = :expr_arg
1604
+
1605
+ assert_lex3("m { 3 }",
1606
+ nil,
1607
+ :tIDENTIFIER, "m", :expr_cmdarg,
1608
+ :tLCURLY, "{", :expr_beg,
1609
+ :tINTEGER, 3, :expr_end,
1610
+ :tRCURLY, "}", :expr_endarg)
1477
1611
  end
1478
1612
 
1479
1613
  def test_yylex_open_curly_bracket_block
1480
- @lex.lex_state = :expr_endarg # seen m(3)
1481
- util_lex_token("{ 4 }",
1482
- :tLBRACE_ARG, "{",
1483
- :tINTEGER, 4,
1484
- :tRCURLY, "}")
1614
+ self.lex_state = :expr_endarg # seen m(3)
1615
+
1616
+ assert_lex3("{ 4 }",
1617
+ nil,
1618
+ :tLBRACE_ARG, "{", :expr_beg,
1619
+ :tINTEGER, 4, :expr_end,
1620
+ :tRCURLY, "}", :expr_endarg)
1485
1621
  end
1486
1622
 
1487
1623
  def test_yylex_open_square_bracket_arg
1488
- @lex.lex_state = :expr_arg
1489
- util_lex_token("m [ 3 ]",
1490
- :tIDENTIFIER, "m",
1491
- :tLBRACK, "[",
1492
- :tINTEGER, 3,
1493
- :tRBRACK, "]")
1624
+ self.lex_state = :expr_arg
1625
+
1626
+ assert_lex3("m [ 3 ]",
1627
+ nil,
1628
+ :tIDENTIFIER, "m", :expr_cmdarg,
1629
+ :tLBRACK, "[", :expr_beg,
1630
+ :tINTEGER, 3, :expr_end,
1631
+ :tRBRACK, "]", :expr_endarg)
1494
1632
  end
1495
1633
 
1496
1634
  def test_yylex_open_square_bracket_ary
1497
- util_lex_token("[1, 2, 3]",
1498
- :tLBRACK, "[",
1499
- :tINTEGER, 1,
1500
- :tCOMMA, ",",
1501
- :tINTEGER, 2,
1502
- :tCOMMA, ",",
1503
- :tINTEGER, 3,
1504
- :tRBRACK, "]")
1635
+ assert_lex3("[1, 2, 3]",
1636
+ nil,
1637
+ :tLBRACK, "[", :expr_beg,
1638
+ :tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1639
+ :tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1640
+ :tINTEGER, 3, :expr_end,
1641
+ :tRBRACK, "]", :expr_endarg)
1505
1642
  end
1506
1643
 
1507
1644
  def test_yylex_open_square_bracket_meth
1508
- util_lex_token("m[3]",
1509
- :tIDENTIFIER, "m",
1510
- :tLBRACK2, "[",
1511
- :tINTEGER, 3,
1512
- :tRBRACK, "]")
1645
+ assert_lex3("m[3]",
1646
+ nil,
1647
+ :tIDENTIFIER, "m", :expr_cmdarg,
1648
+ :tLBRACK2, "[", :expr_beg,
1649
+ :tINTEGER, 3, :expr_end,
1650
+ :tRBRACK, "]", :expr_endarg)
1513
1651
  end
1514
1652
 
1515
1653
  def test_yylex_or
1516
- util_lex_token "|", :tPIPE, "|"
1654
+ assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
1517
1655
  end
1518
1656
 
1519
1657
  def test_yylex_or2
1520
- util_lex_token "||", :tOROP, "||"
1658
+ assert_lex3("||", nil, :tOROP, "||", :expr_beg)
1521
1659
  end
1522
1660
 
1523
1661
  def test_yylex_or2_equals
1524
- util_lex_token "||=", :tOP_ASGN, "||"
1662
+ assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
1525
1663
  end
1526
1664
 
1527
1665
  def test_yylex_or_equals
1528
- util_lex_token "|=", :tOP_ASGN, "|"
1666
+ assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
1529
1667
  end
1530
1668
 
1531
1669
  def test_yylex_percent
1532
- util_lex_token("a % 2",
1533
- :tIDENTIFIER, "a",
1534
- :tPERCENT, "%",
1535
- :tINTEGER, 2)
1670
+ assert_lex3("a % 2",
1671
+ nil,
1672
+ :tIDENTIFIER, "a", :expr_cmdarg,
1673
+ :tPERCENT, "%", :expr_beg,
1674
+ :tINTEGER, 2, :expr_end)
1536
1675
  end
1537
1676
 
1538
1677
  def test_yylex_percent_equals
1539
- util_lex_token("a %= 2",
1540
- :tIDENTIFIER, "a",
1541
- :tOP_ASGN, "%",
1542
- :tINTEGER, 2)
1678
+ assert_lex3("a %= 2",
1679
+ nil,
1680
+ :tIDENTIFIER, "a", :expr_cmdarg,
1681
+ :tOP_ASGN, "%", :expr_beg,
1682
+ :tINTEGER, 2, :expr_end)
1543
1683
  end
1544
1684
 
1545
1685
  def test_yylex_plus
1546
- util_lex_token("1 + 1", # TODO lex_state?
1547
- :tINTEGER, 1,
1548
- :tPLUS, "+",
1549
- :tINTEGER, 1)
1686
+ assert_lex3("1 + 1", # TODO lex_state?
1687
+ nil,
1688
+ :tINTEGER, 1, :expr_end,
1689
+ :tPLUS, "+", :expr_beg,
1690
+ :tINTEGER, 1, :expr_end)
1550
1691
  end
1551
1692
 
1552
1693
  def test_yylex_plus_equals
1553
- util_lex_token "+=", :tOP_ASGN, "+"
1694
+ assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
1554
1695
  end
1555
1696
 
1556
1697
  def test_yylex_plus_method
1557
- @lex.lex_state = :expr_fname
1558
- util_lex_token "+", :tPLUS, "+"
1698
+ self.lex_state = :expr_fname
1699
+
1700
+ assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
1559
1701
  end
1560
1702
 
1561
1703
  def test_yylex_plus_unary_method
1562
- @lex.lex_state = :expr_fname
1563
- util_lex_token "+@", :tUPLUS, "+@"
1704
+ self.lex_state = :expr_fname
1705
+
1706
+ assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
1564
1707
  end
1565
1708
 
1566
1709
  def test_yylex_not_unary_method
1567
- skip "not yet"
1710
+ self.lex_state = :expr_fname
1568
1711
 
1569
- @lex.lex_state = :expr_fname
1570
- util_lex_token "!@", :tUBANG, "!@"
1712
+ assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
1571
1713
  end
1572
1714
 
1573
1715
  def test_yylex_numbers
1574
- util_lex_token "0b10", :tINTEGER, 2
1575
- util_lex_token "0B10", :tINTEGER, 2
1716
+ assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
1717
+ assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
1576
1718
 
1577
- util_lex_token "0d10", :tINTEGER, 10
1578
- util_lex_token "0D10", :tINTEGER, 10
1719
+ assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
1720
+ assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
1579
1721
 
1580
- util_lex_token "0x10", :tINTEGER, 16
1581
- util_lex_token "0X10", :tINTEGER, 16
1722
+ assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
1723
+ assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
1582
1724
 
1583
- util_lex_token "0o10", :tINTEGER, 8
1584
- util_lex_token "0O10", :tINTEGER, 8
1585
- util_lex_token "0o", :tINTEGER, 0
1586
- util_lex_token "0O", :tINTEGER, 0
1725
+ assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
1726
+ assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
1587
1727
 
1588
- util_lex_token "0o", :tINTEGER, 0
1589
- util_lex_token "0O", :tINTEGER, 0
1728
+ assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
1729
+ assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
1590
1730
 
1591
- util_lex_token "0", :tINTEGER, 0
1731
+ assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
1592
1732
 
1593
- util_bad_token "0x"
1594
- util_bad_token "0X"
1595
- util_bad_token "0b"
1596
- util_bad_token "0B"
1597
- util_bad_token "0d"
1598
- util_bad_token "0D"
1733
+ refute_lex "0x"
1734
+ refute_lex "0X"
1735
+ refute_lex "0b"
1736
+ refute_lex "0B"
1737
+ refute_lex "0d"
1738
+ refute_lex "0D"
1599
1739
 
1600
- util_bad_token "08"
1601
- util_bad_token "09"
1602
- util_bad_token "0o8"
1603
- util_bad_token "0o9"
1604
- util_bad_token "0O8"
1605
- util_bad_token "0O9"
1740
+ refute_lex "08"
1741
+ refute_lex "09"
1742
+ refute_lex "0o8"
1743
+ refute_lex "0o9"
1744
+ refute_lex "0O8"
1745
+ refute_lex "0O9"
1606
1746
 
1607
- util_bad_token "1_e1"
1608
- util_bad_token "1_.1"
1609
- util_bad_token "1__1"
1747
+ refute_lex "1_e1"
1748
+ refute_lex "1_.1"
1749
+ refute_lex "1__1"
1610
1750
  end
1611
1751
 
1612
1752
  def test_yylex_plus_unary_number
1613
- util_lex_token("+42",
1614
- :tINTEGER, 42)
1753
+ assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
1615
1754
  end
1616
1755
 
1617
1756
  def test_yylex_question__18
1618
1757
  setup_lexer_class Ruby18Parser
1619
1758
 
1620
- util_lex_token "?*", :tINTEGER, 42
1759
+ assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
1621
1760
  end
1622
1761
 
1623
1762
  def test_yylex_question__19
1624
1763
  setup_lexer_class Ruby19Parser
1625
1764
 
1626
- util_lex_token "?*", :tSTRING, "*"
1765
+ assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
1627
1766
  end
1628
1767
 
1629
1768
  def test_yylex_question_bad_eos
1630
- util_bad_token "?"
1769
+ refute_lex "?"
1631
1770
  end
1632
1771
 
1633
1772
  def test_yylex_question_ws
1634
- util_lex_token "? ", :tEH, "?"
1635
- util_lex_token "?\n", :tEH, "?"
1636
- util_lex_token "?\t", :tEH, "?"
1637
- util_lex_token "?\v", :tEH, "?"
1638
- util_lex_token "?\r", :tEH, "?"
1639
- util_lex_token "?\f", :tEH, "?"
1773
+ assert_lex3("? ", nil, :tEH, "?", :expr_value)
1774
+ assert_lex3("?\n", nil, :tEH, "?", :expr_value)
1775
+ assert_lex3("?\t", nil, :tEH, "?", :expr_value)
1776
+ assert_lex3("?\v", nil, :tEH, "?", :expr_value)
1777
+ assert_lex3("?\r", nil, :tEH, "?", :expr_value)
1778
+ assert_lex3("?\f", nil, :tEH, "?", :expr_value)
1640
1779
  end
1641
1780
 
1642
1781
  def test_yylex_question_ws_backslashed__18
1643
1782
  setup_lexer_class Ruby18Parser
1644
1783
 
1645
- @lex.lex_state = :expr_beg
1646
- util_lex_token "?\\ ", :tINTEGER, 32
1647
- @lex.lex_state = :expr_beg
1648
- util_lex_token "?\\n", :tINTEGER, 10
1649
- @lex.lex_state = :expr_beg
1650
- util_lex_token "?\\t", :tINTEGER, 9
1651
- @lex.lex_state = :expr_beg
1652
- util_lex_token "?\\v", :tINTEGER, 11
1653
- @lex.lex_state = :expr_beg
1654
- util_lex_token "?\\r", :tINTEGER, 13
1655
- @lex.lex_state = :expr_beg
1656
- util_lex_token "?\\f", :tINTEGER, 12
1784
+ assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
1785
+ assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
1786
+ assert_lex3("?\\t", nil, :tINTEGER, 9, :expr_end)
1787
+ assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
1788
+ assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
1789
+ assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
1657
1790
  end
1658
1791
 
1659
1792
  def test_yylex_question_ws_backslashed__19
1660
1793
  setup_lexer_class Ruby19Parser
1661
1794
 
1662
- @lex.lex_state = :expr_beg
1663
- util_lex_token "?\\ ", :tSTRING, " "
1664
- @lex.lex_state = :expr_beg
1665
- util_lex_token "?\\n", :tSTRING, "\n"
1666
- @lex.lex_state = :expr_beg
1667
- util_lex_token "?\\t", :tSTRING, "\t"
1668
- @lex.lex_state = :expr_beg
1669
- util_lex_token "?\\v", :tSTRING, "\v"
1670
- @lex.lex_state = :expr_beg
1671
- util_lex_token "?\\r", :tSTRING, "\r"
1672
- @lex.lex_state = :expr_beg
1673
- util_lex_token "?\\f", :tSTRING, "\f"
1795
+ assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
1796
+ assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
1797
+ assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
1798
+ assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
1799
+ assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
1800
+ assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
1674
1801
  end
1675
1802
 
1676
1803
  def test_yylex_rbracket
1677
- util_lex_token "]", :tRBRACK, "]"
1804
+ assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
1678
1805
  end
1679
1806
 
1680
1807
  def test_yylex_rcurly
1681
- util_lex_token "}", :tRCURLY, "}"
1808
+ assert_lex3("}", nil, :tRCURLY, "}", :expr_endarg)
1682
1809
  end
1683
1810
 
1684
1811
  def test_yylex_regexp
1685
- util_lex_token("/regexp/",
1686
- :tREGEXP_BEG, "/",
1687
- :tSTRING_CONTENT, "regexp",
1688
- :tREGEXP_END, "")
1812
+ assert_lex3("/regexp/",
1813
+ nil,
1814
+ :tREGEXP_BEG, "/", :expr_beg,
1815
+ :tSTRING_CONTENT, "regexp", :expr_beg,
1816
+ :tREGEXP_END, "", :expr_end)
1689
1817
  end
1690
1818
 
1691
1819
  def test_yylex_regexp_ambiguous
1692
- util_lex_token("method /regexp/",
1693
- :tIDENTIFIER, "method",
1694
- :tREGEXP_BEG, "/",
1695
- :tSTRING_CONTENT, "regexp",
1696
- :tREGEXP_END, "")
1820
+ assert_lex3("method /regexp/",
1821
+ nil,
1822
+ :tIDENTIFIER, "method", :expr_cmdarg,
1823
+ :tREGEXP_BEG, "/", :expr_cmdarg,
1824
+ :tSTRING_CONTENT, "regexp", :expr_cmdarg,
1825
+ :tREGEXP_END, "", :expr_end)
1697
1826
  end
1698
1827
 
1699
1828
  def test_yylex_regexp_bad
1700
- util_bad_token("/.*/xyz",
1701
- :tREGEXP_BEG, "/",
1702
- :tSTRING_CONTENT, ".*")
1829
+ refute_lex("/.*/xyz",
1830
+ :tREGEXP_BEG, "/",
1831
+ :tSTRING_CONTENT, ".*")
1703
1832
  end
1704
1833
 
1705
1834
  def test_yylex_regexp_escape_C
1706
- util_lex_token('/regex\\C-x/',
1707
- :tREGEXP_BEG, "/",
1708
- :tSTRING_CONTENT, "regex\\C-x",
1709
- :tREGEXP_END, "")
1835
+ assert_lex3("/regex\\C-x/",
1836
+ nil,
1837
+ :tREGEXP_BEG, "/", :expr_beg,
1838
+ :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
1839
+ :tREGEXP_END, "", :expr_end)
1710
1840
  end
1711
1841
 
1712
1842
  def test_yylex_regexp_escape_C_M
1713
- util_lex_token('/regex\\C-\\M-x/',
1714
- :tREGEXP_BEG, "/",
1715
- :tSTRING_CONTENT, "regex\\C-\\M-x",
1716
- :tREGEXP_END, "")
1843
+ assert_lex3("/regex\\C-\\M-x/",
1844
+ nil,
1845
+ :tREGEXP_BEG, "/", :expr_beg,
1846
+ :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1847
+ :tREGEXP_END, "", :expr_end)
1717
1848
  end
1718
1849
 
1719
1850
  def test_yylex_regexp_escape_C_M_craaaazy
1720
- util_lex_token("/regex\\C-\\\n\\M-x/",
1721
- :tREGEXP_BEG, "/",
1722
- :tSTRING_CONTENT, "regex\\C-\\M-x",
1723
- :tREGEXP_END, "")
1851
+ assert_lex3("/regex\\C-\\\n\\M-x/",
1852
+ nil,
1853
+ :tREGEXP_BEG, "/", :expr_beg,
1854
+ :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1855
+ :tREGEXP_END, "", :expr_end)
1724
1856
  end
1725
1857
 
1726
1858
  def test_yylex_regexp_escape_C_bad_dash
1727
- util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
1859
+ refute_lex '/regex\\Cx/', :tREGEXP_BEG, "/"
1728
1860
  end
1729
1861
 
1730
1862
  def test_yylex_regexp_escape_C_bad_dash_eos
1731
- util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
1863
+ refute_lex '/regex\\C-/', :tREGEXP_BEG, "/"
1732
1864
  end
1733
1865
 
1734
1866
  def test_yylex_regexp_escape_C_bad_dash_eos2
1735
- util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
1867
+ refute_lex '/regex\\C-', :tREGEXP_BEG, "/"
1736
1868
  end
1737
1869
 
1738
1870
  def test_yylex_regexp_escape_C_bad_eos
1739
- util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
1871
+ refute_lex '/regex\\C/', :tREGEXP_BEG, "/"
1740
1872
  end
1741
1873
 
1742
1874
  def test_yylex_regexp_escape_C_bad_eos2
1743
- util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
1875
+ refute_lex '/regex\\c', :tREGEXP_BEG, "/"
1744
1876
  end
1745
1877
 
1746
1878
  def test_yylex_regexp_escape_M
1747
- util_lex_token('/regex\\M-x/',
1748
- :tREGEXP_BEG, "/",
1749
- :tSTRING_CONTENT, "regex\\M-x",
1750
- :tREGEXP_END, "")
1879
+ assert_lex3("/regex\\M-x/",
1880
+ nil,
1881
+ :tREGEXP_BEG, "/", :expr_beg,
1882
+ :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
1883
+ :tREGEXP_END, "", :expr_end)
1751
1884
  end
1752
1885
 
1753
1886
  def test_yylex_regexp_escape_M_C
1754
- util_lex_token('/regex\\M-\\C-x/',
1755
- :tREGEXP_BEG, "/",
1756
- :tSTRING_CONTENT, "regex\\M-\\C-x",
1757
- :tREGEXP_END, "")
1887
+ assert_lex3("/regex\\M-\\C-x/",
1888
+ nil,
1889
+ :tREGEXP_BEG, "/", :expr_beg,
1890
+ :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
1891
+ :tREGEXP_END, "", :expr_end)
1758
1892
  end
1759
1893
 
1760
1894
  def test_yylex_regexp_escape_M_bad_dash
1761
- util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
1895
+ refute_lex '/regex\\Mx/', :tREGEXP_BEG, "/"
1762
1896
  end
1763
1897
 
1764
1898
  def test_yylex_regexp_escape_M_bad_dash_eos
1765
- util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
1899
+ refute_lex '/regex\\M-/', :tREGEXP_BEG, "/"
1766
1900
  end
1767
1901
 
1768
1902
  def test_yylex_regexp_escape_M_bad_dash_eos2
1769
- util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
1903
+ refute_lex '/regex\\M-', :tREGEXP_BEG, "/"
1770
1904
  end
1771
1905
 
1772
1906
  def test_yylex_regexp_escape_M_bad_eos
1773
- util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
1907
+ refute_lex '/regex\\M/', :tREGEXP_BEG, "/"
1774
1908
  end
1775
1909
 
1776
1910
  def test_yylex_regexp_escape_backslash_slash
1777
- util_lex_token('/\\//',
1778
- :tREGEXP_BEG, "/",
1779
- :tSTRING_CONTENT, '\\/',
1780
- :tREGEXP_END, "")
1911
+ assert_lex3("/\\//",
1912
+ nil,
1913
+ :tREGEXP_BEG, "/", :expr_beg,
1914
+ :tSTRING_CONTENT, "\\/", :expr_beg,
1915
+ :tREGEXP_END, "", :expr_end)
1781
1916
  end
1782
1917
 
1783
1918
  def test_yylex_regexp_escape_backslash_terminator
1784
- util_lex_token('%r%blah\\%blah%',
1785
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1786
- :tSTRING_CONTENT, "blah\\%blah",
1787
- :tREGEXP_END, "")
1919
+ assert_lex3("%r%blah\\%blah%",
1920
+ nil,
1921
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1922
+ :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
1923
+ :tREGEXP_END, "", :expr_end)
1788
1924
  end
1789
1925
 
1790
1926
  def test_yylex_regexp_escape_backslash_terminator_meta1
1791
- util_lex_token('%r{blah\\}blah}',
1792
- :tREGEXP_BEG, "%r{", # FIX ?!?
1793
- :tSTRING_CONTENT, "blah\\}blah",
1794
- :tREGEXP_END, "")
1927
+ assert_lex3("%r{blah\\}blah}",
1928
+ nil,
1929
+ :tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
1930
+ :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
1931
+ :tREGEXP_END, "", :expr_end)
1795
1932
  end
1796
1933
 
1797
1934
  def test_yylex_regexp_escape_backslash_terminator_meta2
1798
- util_lex_token('%r/blah\\/blah/',
1799
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1800
- :tSTRING_CONTENT, "blah\\/blah",
1801
- :tREGEXP_END, "")
1935
+ assert_lex3("%r/blah\\/blah/",
1936
+ nil,
1937
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1938
+ :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
1939
+ :tREGEXP_END, "", :expr_end)
1802
1940
  end
1803
1941
 
1804
1942
  def test_yylex_regexp_escape_backslash_terminator_meta3
1805
- util_lex_token('%r/blah\\%blah/',
1806
- :tREGEXP_BEG, "%r\000", # FIX ?!?
1807
- :tSTRING_CONTENT, "blah\\%blah",
1808
- :tREGEXP_END, "")
1943
+ assert_lex3("%r/blah\\%blah/",
1944
+ nil,
1945
+ :tREGEXP_BEG, "%r\000", :expr_beg,
1946
+ :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
1947
+ :tREGEXP_END, "", :expr_end)
1809
1948
  end
1810
1949
 
1811
1950
  def test_yylex_regexp_escape_bad_eos
1812
- util_bad_token '/regex\\', :tREGEXP_BEG, "/"
1951
+ refute_lex '/regex\\', :tREGEXP_BEG, "/"
1813
1952
  end
1814
1953
 
1815
1954
  def test_yylex_regexp_escape_bs
1816
- util_lex_token('/regex\\\\regex/',
1817
- :tREGEXP_BEG, "/",
1818
- :tSTRING_CONTENT, "regex\\\\regex",
1819
- :tREGEXP_END, "")
1955
+ assert_lex3("/regex\\\\regex/",
1956
+ nil,
1957
+ :tREGEXP_BEG, "/", :expr_beg,
1958
+ :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
1959
+ :tREGEXP_END, "", :expr_end)
1820
1960
  end
1821
1961
 
1822
1962
  def test_yylex_regexp_escape_c
1823
- util_lex_token('/regex\\cxxx/',
1824
- :tREGEXP_BEG, "/",
1825
- :tSTRING_CONTENT, "regex\\cxxx",
1826
- :tREGEXP_END, "")
1963
+ assert_lex3("/regex\\cxxx/",
1964
+ nil,
1965
+ :tREGEXP_BEG, "/", :expr_beg,
1966
+ :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
1967
+ :tREGEXP_END, "", :expr_end)
1827
1968
  end
1828
1969
 
1829
1970
  def test_yylex_regexp_escape_c_backslash
1830
- util_lex_token('/regex\\c\\n/',
1831
- :tREGEXP_BEG, "/",
1832
- :tSTRING_CONTENT, "regex\\c\\n",
1833
- :tREGEXP_END, "")
1971
+ assert_lex3("/regex\\c\\n/",
1972
+ nil,
1973
+ :tREGEXP_BEG, "/", :expr_beg,
1974
+ :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
1975
+ :tREGEXP_END, "", :expr_end)
1834
1976
  end
1835
1977
 
1836
1978
  def test_yylex_regexp_escape_chars
1837
- util_lex_token('/re\\tge\\nxp/',
1838
- :tREGEXP_BEG, "/",
1839
- :tSTRING_CONTENT, "re\\tge\\nxp",
1840
- :tREGEXP_END, "")
1979
+ assert_lex3("/re\\tge\\nxp/",
1980
+ nil,
1981
+ :tREGEXP_BEG, "/", :expr_beg,
1982
+ :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
1983
+ :tREGEXP_END, "", :expr_end)
1841
1984
  end
1842
1985
 
1843
1986
  def test_yylex_regexp_escape_double_backslash
1844
1987
  regexp = '/[\\/\\\\]$/'
1845
- util_lex_token(regexp,
1846
- :tREGEXP_BEG, "/",
1847
- :tSTRING_CONTENT, regexp[1..-2],
1848
- :tREGEXP_END, "")
1988
+ assert_lex3(regexp.dup,
1989
+ nil,
1990
+ :tREGEXP_BEG, "/", :expr_beg,
1991
+ :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
1992
+ :tREGEXP_END, "", :expr_end)
1849
1993
  end
1850
1994
 
1851
1995
  def test_yylex_regexp_escape_hex
1852
- util_lex_token('/regex\\x61xp/',
1853
- :tREGEXP_BEG, "/",
1854
- :tSTRING_CONTENT, "regex\\x61xp",
1855
- :tREGEXP_END, "")
1996
+ assert_lex3("/regex\\x61xp/",
1997
+ nil,
1998
+ :tREGEXP_BEG, "/", :expr_beg,
1999
+ :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
2000
+ :tREGEXP_END, "", :expr_end)
1856
2001
  end
1857
2002
 
1858
2003
  def test_yylex_regexp_escape_hex_bad
1859
- util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
2004
+ refute_lex '/regex\\xzxp/', :tREGEXP_BEG, "/"
1860
2005
  end
1861
2006
 
1862
2007
  def test_yylex_regexp_escape_hex_one
1863
- util_lex_token('/^[\\xd\\xa]{2}/on',
1864
- :tREGEXP_BEG, '/',
1865
- :tSTRING_CONTENT, '^[\\xd\\xa]{2}',
1866
- :tREGEXP_END, 'on')
2008
+ assert_lex3("/^[\\xd\\xa]{2}/on",
2009
+ nil,
2010
+ :tREGEXP_BEG, "/", :expr_beg,
2011
+ :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
2012
+ :tREGEXP_END, "on", :expr_end)
1867
2013
  end
1868
2014
 
1869
2015
  def test_yylex_regexp_escape_oct1
1870
- util_lex_token('/regex\\0xp/',
1871
- :tREGEXP_BEG, "/",
1872
- :tSTRING_CONTENT, "regex\\0xp",
1873
- :tREGEXP_END, "")
2016
+ assert_lex3("/regex\\0xp/",
2017
+ nil,
2018
+ :tREGEXP_BEG, "/", :expr_beg,
2019
+ :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
2020
+ :tREGEXP_END, "", :expr_end)
1874
2021
  end
1875
2022
 
1876
2023
  def test_yylex_regexp_escape_oct2
1877
- util_lex_token('/regex\\07xp/',
1878
- :tREGEXP_BEG, "/",
1879
- :tSTRING_CONTENT, "regex\\07xp",
1880
- :tREGEXP_END, "")
2024
+ assert_lex3("/regex\\07xp/",
2025
+ nil,
2026
+ :tREGEXP_BEG, "/", :expr_beg,
2027
+ :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
2028
+ :tREGEXP_END, "", :expr_end)
1881
2029
  end
1882
2030
 
1883
2031
  def test_yylex_regexp_escape_oct3
1884
- util_lex_token('/regex\\10142/',
1885
- :tREGEXP_BEG, "/",
1886
- :tSTRING_CONTENT, "regex\\10142",
1887
- :tREGEXP_END, "")
2032
+ assert_lex3("/regex\\10142/",
2033
+ nil,
2034
+ :tREGEXP_BEG, "/", :expr_beg,
2035
+ :tSTRING_CONTENT, "regex\\10142", :expr_beg,
2036
+ :tREGEXP_END, "", :expr_end)
1888
2037
  end
1889
2038
 
1890
2039
  def test_yylex_regexp_escape_return
1891
- util_lex_token("/regex\\\nregex/",
1892
- :tREGEXP_BEG, "/",
1893
- :tSTRING_CONTENT, "regexregex",
1894
- :tREGEXP_END, "")
2040
+ assert_lex3("/regex\\\nregex/",
2041
+ nil,
2042
+ :tREGEXP_BEG, "/", :expr_beg,
2043
+ :tSTRING_CONTENT, "regexregex", :expr_beg,
2044
+ :tREGEXP_END, "", :expr_end)
1895
2045
  end
1896
2046
 
1897
2047
  def test_yylex_regexp_nm
1898
- util_lex_token("/.*/nm",
1899
- :tREGEXP_BEG, "/",
1900
- :tSTRING_CONTENT, ".*",
1901
- :tREGEXP_END, "nm")
2048
+ assert_lex3("/.*/nm",
2049
+ nil,
2050
+ :tREGEXP_BEG, "/", :expr_beg,
2051
+ :tSTRING_CONTENT, ".*", :expr_beg,
2052
+ :tREGEXP_END, "nm", :expr_end)
1902
2053
  end
1903
2054
 
1904
2055
  def test_yylex_rparen
1905
- util_lex_token ")", :tRPAREN, ")"
2056
+ assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
1906
2057
  end
1907
2058
 
1908
2059
  def test_yylex_rshft
1909
- util_lex_token("a >> 2",
1910
- :tIDENTIFIER, "a",
1911
- :tRSHFT, ">>",
1912
- :tINTEGER, 2)
2060
+ assert_lex3("a >> 2",
2061
+ nil,
2062
+ :tIDENTIFIER, "a", :expr_cmdarg,
2063
+ :tRSHFT, ">>", :expr_beg,
2064
+ :tINTEGER, 2, :expr_end)
1913
2065
  end
1914
2066
 
1915
2067
  def test_yylex_rshft_equals
1916
- util_lex_token("a >>= 2",
1917
- :tIDENTIFIER, "a",
1918
- :tOP_ASGN, ">>",
1919
- :tINTEGER, 2)
2068
+ assert_lex3("a >>= 2",
2069
+ nil,
2070
+ :tIDENTIFIER, "a", :expr_cmdarg,
2071
+ :tOP_ASGN, ">>", :expr_beg,
2072
+ :tINTEGER, 2, :expr_end)
1920
2073
  end
1921
2074
 
1922
2075
  def test_yylex_star
1923
- util_lex_token("a * ",
1924
- :tIDENTIFIER, "a",
1925
- :tSTAR2, "*")
1926
-
1927
- assert_equal :expr_beg, @lex.lex_state
2076
+ assert_lex3("a * ",
2077
+ nil,
2078
+ :tIDENTIFIER, "a", :expr_cmdarg,
2079
+ :tSTAR2, "*", :expr_beg)
1928
2080
  end
1929
2081
 
1930
2082
  def test_yylex_star2
1931
- util_lex_token("a ** ",
1932
- :tIDENTIFIER, "a",
1933
- :tPOW, "**")
1934
-
1935
- assert_equal :expr_beg, @lex.lex_state
2083
+ assert_lex3("a ** ",
2084
+ nil,
2085
+ :tIDENTIFIER, "a", :expr_cmdarg,
2086
+ :tPOW, "**", :expr_beg)
1936
2087
  end
1937
2088
 
1938
2089
  def test_yylex_star2_equals
1939
- util_lex_token("a **= ",
1940
- :tIDENTIFIER, "a",
1941
- :tOP_ASGN, "**")
1942
-
1943
- assert_equal :expr_beg, @lex.lex_state
2090
+ assert_lex3("a **= ",
2091
+ nil,
2092
+ :tIDENTIFIER, "a", :expr_cmdarg,
2093
+ :tOP_ASGN, "**", :expr_beg)
1944
2094
  end
1945
2095
 
1946
2096
  def test_yylex_star_arg
1947
- @lex.lex_state = :expr_arg
1948
-
1949
- util_lex_token(" *a",
1950
- :tSTAR, "*",
1951
- :tIDENTIFIER, "a")
2097
+ self.lex_state = :expr_arg
1952
2098
 
1953
- assert_equal :expr_arg, @lex.lex_state
2099
+ assert_lex3(" *a",
2100
+ nil,
2101
+ :tSTAR, "*", :expr_beg,
2102
+ :tIDENTIFIER, "a", :expr_arg)
1954
2103
  end
1955
2104
 
1956
2105
  def test_yylex_star_arg_beg
1957
- @lex.lex_state = :expr_beg
2106
+ self.lex_state = :expr_beg
1958
2107
 
1959
- util_lex_token("*a",
1960
- :tSTAR, "*",
1961
- :tIDENTIFIER, "a")
1962
-
1963
- assert_equal :expr_arg, @lex.lex_state
2108
+ assert_lex3("*a",
2109
+ nil,
2110
+ :tSTAR, "*", :expr_beg,
2111
+ :tIDENTIFIER, "a", :expr_arg)
1964
2112
  end
1965
2113
 
1966
2114
  def test_yylex_star_arg_beg_fname
1967
- @lex.lex_state = :expr_fname
2115
+ self.lex_state = :expr_fname
2116
+
2117
+ assert_lex3("*a",
2118
+ nil,
2119
+ :tSTAR2, "*", :expr_arg,
2120
+ :tIDENTIFIER, "a", :expr_arg)
2121
+ end
1968
2122
 
1969
- util_lex_token("*a",
1970
- :tSTAR2, "*",
1971
- :tIDENTIFIER, "a")
2123
+ def test_yylex_star_arg_beg_fname2
2124
+ self.lex_state = :expr_fname
1972
2125
 
1973
- assert_equal :expr_arg, @lex.lex_state
2126
+ assert_lex3("*a",
2127
+ nil,
2128
+ :tSTAR2, "*", :expr_arg,
2129
+ :tIDENTIFIER, "a", :expr_arg)
1974
2130
  end
1975
2131
 
1976
2132
  def test_yylex_star_equals
1977
- util_lex_token("a *= ",
1978
- :tIDENTIFIER, "a",
1979
- :tOP_ASGN, "*")
1980
-
1981
- assert_equal :expr_beg, @lex.lex_state
2133
+ assert_lex3("a *= ",
2134
+ nil,
2135
+ :tIDENTIFIER, "a", :expr_cmdarg,
2136
+ :tOP_ASGN, "*", :expr_beg)
1982
2137
  end
1983
2138
 
1984
2139
  def test_yylex_string_bad_eos
1985
- util_bad_token('%',
1986
- :tSTRING_BEG, '%')
2140
+ refute_lex('%', :tSTRING_BEG, '%')
1987
2141
  end
1988
2142
 
1989
2143
  def test_yylex_string_bad_eos_quote
1990
- util_bad_token('%{nest',
1991
- :tSTRING_BEG, '%}')
2144
+ refute_lex('%{nest', :tSTRING_BEG, '%}')
1992
2145
  end
1993
2146
 
1994
2147
  def test_yylex_string_double
1995
- util_lex_token('"string"',
1996
- :tSTRING, "string")
2148
+ assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
1997
2149
  end
1998
2150
 
1999
2151
  def test_yylex_string_double_escape_C
2000
- util_lex_token('"\\C-a"',
2001
- :tSTRING, "\001")
2152
+ assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
2002
2153
  end
2003
2154
 
2004
2155
  def test_yylex_string_double_escape_C_backslash
2005
- util_lex_token('"\\C-\\\\"',
2006
- :tSTRING_BEG, "\"",
2007
- :tSTRING_CONTENT, "\034",
2008
- :tSTRING_END, "\"")
2156
+ assert_lex3("\"\\C-\\\\\"",
2157
+ nil,
2158
+ :tSTRING_BEG, "\"", :expr_beg,
2159
+ :tSTRING_CONTENT, "\034", :expr_beg,
2160
+ :tSTRING_END, "\"", :expr_end)
2009
2161
  end
2010
2162
 
2011
2163
  def test_yylex_string_double_escape_C_escape
2012
- util_lex_token('"\\C-\\M-a"',
2013
- :tSTRING_BEG, "\"",
2014
- :tSTRING_CONTENT, "\201",
2015
- :tSTRING_END, "\"")
2164
+ assert_lex3("\"\\C-\\M-a\"",
2165
+ nil,
2166
+ :tSTRING_BEG, "\"", :expr_beg,
2167
+ :tSTRING_CONTENT, "\201", :expr_beg,
2168
+ :tSTRING_END, "\"", :expr_end)
2016
2169
  end
2017
2170
 
2018
2171
  def test_yylex_string_double_escape_C_question
2019
- util_lex_token('"\\C-?"',
2020
- :tSTRING, "\177")
2172
+ assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
2173
+ end
2174
+
2175
+ def test_yylex_string_utf8_simple
2176
+ chr = [0x3024].pack("U")
2177
+
2178
+ assert_lex3('"\u{3024}"',
2179
+ s(:str, chr),
2180
+ :tSTRING, chr, :expr_end)
2181
+ end
2182
+
2183
+ def test_yylex_string_utf8_complex
2184
+ chr = [0x3024].pack("U")
2185
+
2186
+ assert_lex3('"#@a\u{3024}"',
2187
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2188
+ :tSTRING_BEG, '"', :expr_beg,
2189
+ :tSTRING_DVAR, nil, :expr_beg,
2190
+ :tSTRING_CONTENT, "@a"+chr, :expr_beg,
2191
+ :tSTRING_END, '"', :expr_end)
2021
2192
  end
2022
2193
 
2023
2194
  def test_yylex_string_double_escape_M
2024
2195
  chr = "\341"
2025
2196
  chr.force_encoding("UTF-8") if RubyLexer::RUBY19
2026
2197
 
2027
- util_lex_token('"\\M-a"',
2028
- :tSTRING, chr)
2198
+ assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
2029
2199
  end
2030
2200
 
2031
2201
  def test_why_does_ruby_hate_me?
2032
- util_lex_token('"Nl%\000\000A\000\999"', # you should be ashamed
2033
- :tSTRING,
2034
- ["Nl%", "\x00", "\x00", "A", "\x00", "999"].join)
2202
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2203
+ nil,
2204
+ :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
2035
2205
  end
2036
2206
 
2037
2207
  def test_yylex_string_double_escape_M_backslash
2038
- util_lex_token('"\\M-\\\\"',
2039
- :tSTRING_BEG, "\"",
2040
- :tSTRING_CONTENT, "\334",
2041
- :tSTRING_END, "\"")
2208
+ assert_lex3("\"\\M-\\\\\"",
2209
+ nil,
2210
+ :tSTRING_BEG, "\"", :expr_beg,
2211
+ :tSTRING_CONTENT, "\334", :expr_beg,
2212
+ :tSTRING_END, "\"", :expr_end)
2042
2213
  end
2043
2214
 
2044
2215
  def test_yylex_string_double_escape_M_escape
2045
- util_lex_token('"\\M-\\C-a"',
2046
- :tSTRING_BEG, "\"",
2047
- :tSTRING_CONTENT, "\201",
2048
- :tSTRING_END, "\"")
2216
+ assert_lex3("\"\\M-\\C-a\"",
2217
+ nil,
2218
+ :tSTRING_BEG, "\"", :expr_beg,
2219
+ :tSTRING_CONTENT, "\201", :expr_beg,
2220
+ :tSTRING_END, "\"", :expr_end)
2049
2221
  end
2050
2222
 
2051
2223
  def test_yylex_string_double_escape_bs1
2052
- util_lex_token('"a\\a\\a"',
2053
- :tSTRING, "a\a\a")
2224
+ assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
2054
2225
  end
2055
2226
 
2056
2227
  def test_yylex_string_double_escape_bs2
2057
- util_lex_token('"a\\\\a"',
2058
- :tSTRING, "a\\a")
2228
+ assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
2059
2229
  end
2060
2230
 
2061
2231
  def test_yylex_string_double_escape_c
2062
- util_lex_token('"\\ca"',
2063
- :tSTRING, "\001")
2232
+ assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
2064
2233
  end
2065
2234
 
2066
2235
  def test_yylex_string_double_escape_c_backslash
2067
- util_lex_token('"\\c\\"',
2068
- :tSTRING_BEG, "\"",
2069
- :tSTRING_CONTENT, "\034",
2070
- :tSTRING_END, "\"")
2236
+ assert_lex3("\"\\c\\\"",
2237
+ nil,
2238
+ :tSTRING_BEG, "\"", :expr_beg,
2239
+ :tSTRING_CONTENT, "\034", :expr_beg,
2240
+ :tSTRING_END, "\"", :expr_end)
2071
2241
  end
2072
2242
 
2073
2243
  def test_yylex_string_double_escape_c_escape
2074
- util_lex_token('"\\c\\M-a"',
2075
- :tSTRING_BEG, "\"",
2076
- :tSTRING_CONTENT, "\201",
2077
- :tSTRING_END, "\"")
2244
+ assert_lex3("\"\\c\\M-a\"",
2245
+ nil,
2246
+ :tSTRING_BEG, "\"", :expr_beg,
2247
+ :tSTRING_CONTENT, "\201", :expr_beg,
2248
+ :tSTRING_END, "\"", :expr_end)
2078
2249
  end
2079
2250
 
2080
2251
  def test_yylex_string_double_escape_c_question
2081
- util_lex_token('"\\c?"',
2082
- :tSTRING, "\177")
2252
+ assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
2083
2253
  end
2084
2254
 
2085
2255
  def test_yylex_string_double_escape_chars
2086
- util_lex_token('"s\\tri\\ng"',
2087
- :tSTRING, "s\tri\ng")
2256
+ assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
2088
2257
  end
2089
2258
 
2090
2259
  def test_yylex_string_double_escape_hex
2091
- util_lex_token('"n = \\x61\\x62\\x63"',
2092
- :tSTRING, "n = abc")
2260
+ assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
2093
2261
  end
2094
2262
 
2095
2263
  def test_yylex_string_double_escape_octal
2096
- util_lex_token('"n = \\101\\102\\103"',
2097
- :tSTRING, "n = ABC")
2264
+ assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
2098
2265
  end
2099
2266
 
2100
2267
  def test_yylex_string_double_escape_octal_fucked
2101
- util_lex_token('"n = \\444"',
2102
- :tSTRING, "n = $")
2268
+ assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
2103
2269
  end
2104
2270
 
2105
2271
  def test_yylex_string_double_interp
2106
- util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
2107
- :tSTRING_BEG, "\"",
2108
- :tSTRING_CONTENT, "blah #x a ",
2109
- :tSTRING_DVAR, nil,
2110
- :tSTRING_CONTENT, "@a b ",
2111
- :tSTRING_DVAR, nil,
2112
- :tSTRING_CONTENT, "$b c ",
2113
- :tSTRING_DBEG, nil,
2114
- :tSTRING_CONTENT, "3} # ",
2115
- :tSTRING_END, "\"")
2272
+ assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
2273
+ nil,
2274
+ :tSTRING_BEG, "\"", :expr_beg,
2275
+ :tSTRING_CONTENT, "blah #x a ", :expr_beg,
2276
+ :tSTRING_DVAR, nil, :expr_beg,
2277
+ :tSTRING_CONTENT, "@a b ", :expr_beg,
2278
+ :tSTRING_DVAR, nil, :expr_beg,
2279
+ :tSTRING_CONTENT, "$b c ", :expr_beg,
2280
+ :tSTRING_DBEG, nil, :expr_beg,
2281
+ :tSTRING_CONTENT, "3} # ", :expr_beg,
2282
+ :tSTRING_END, "\"", :expr_end)
2116
2283
  end
2117
2284
 
2118
2285
  def test_yylex_string_double_nested_curlies
2119
- util_lex_token('%{nest{one{two}one}nest}',
2120
- :tSTRING_BEG, '%}',
2121
- :tSTRING_CONTENT, "nest{one{two}one}nest",
2122
- :tSTRING_END, '}')
2286
+ assert_lex3("%{nest{one{two}one}nest}",
2287
+ nil,
2288
+ :tSTRING_BEG, "%}", :expr_beg,
2289
+ :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
2290
+ :tSTRING_END, "}", :expr_end)
2123
2291
  end
2124
2292
 
2125
2293
  def test_yylex_string_double_no_interp
2126
- util_lex_token("\"# blah\"", # pound first
2127
- :tSTRING, "# blah")
2128
-
2129
- util_lex_token("\"blah # blah\"", # pound not first
2130
- :tSTRING, "blah # blah")
2294
+ assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
2295
+ assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
2131
2296
  end
2132
2297
 
2133
2298
  def test_yylex_string_escape_x_single
2134
- util_lex_token('"\\x0"',
2135
- :tSTRING, "\000")
2299
+ assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
2136
2300
  end
2137
2301
 
2138
2302
  def test_yylex_string_pct_i
2139
- util_lex_token("%i[s1 s2\ns3]",
2140
- :tQSYMBOLS_BEG, "%i[",
2141
- :tSTRING_CONTENT, "s1",
2142
- :tSPACE, nil,
2143
- :tSTRING_CONTENT, "s2",
2144
- :tSPACE, nil,
2145
- :tSTRING_CONTENT, "s3",
2146
- :tSPACE, nil,
2147
- :tSTRING_END, nil)
2303
+ assert_lex3("%i[s1 s2\ns3]",
2304
+ nil,
2305
+ :tQSYMBOLS_BEG, "%i[", :expr_beg,
2306
+ :tSTRING_CONTENT, "s1", :expr_beg,
2307
+ :tSPACE, nil, :expr_beg,
2308
+ :tSTRING_CONTENT, "s2", :expr_beg,
2309
+ :tSPACE, nil, :expr_beg,
2310
+ :tSTRING_CONTENT, "s3", :expr_beg,
2311
+ :tSPACE, nil, :expr_beg,
2312
+ :tSTRING_END, nil, :expr_end)
2148
2313
  end
2149
2314
 
2150
2315
  def test_yylex_string_pct_I
2151
- util_lex_token("%I[s1 s2\ns3]",
2152
- :tSYMBOLS_BEG, "%I[",
2153
- :tSTRING_CONTENT, "s1",
2154
- :tSPACE, nil,
2155
- :tSTRING_CONTENT, "s2",
2156
- :tSPACE, nil,
2157
- :tSTRING_CONTENT, "s3",
2158
- :tSPACE, nil,
2159
- :tSTRING_END, nil)
2316
+ assert_lex3("%I[s1 s2\ns3]",
2317
+ nil,
2318
+ :tSYMBOLS_BEG, "%I[", :expr_beg,
2319
+ :tSTRING_CONTENT, "s1", :expr_beg,
2320
+ :tSPACE, nil, :expr_beg,
2321
+ :tSTRING_CONTENT, "s2", :expr_beg,
2322
+ :tSPACE, nil, :expr_beg,
2323
+ :tSTRING_CONTENT, "s3", :expr_beg,
2324
+ :tSPACE, nil, :expr_beg,
2325
+ :tSTRING_END, nil, :expr_end)
2326
+ end
2327
+
2328
+ def test_yylex_string_pct_i_extra_space
2329
+ assert_lex3("%i[ s1 s2\ns3 ]",
2330
+ nil,
2331
+ :tQSYMBOLS_BEG, "%i[", :expr_beg,
2332
+ :tSTRING_CONTENT, "s1", :expr_beg,
2333
+ :tSPACE, nil, :expr_beg,
2334
+ :tSTRING_CONTENT, "s2", :expr_beg,
2335
+ :tSPACE, nil, :expr_beg,
2336
+ :tSTRING_CONTENT, "s3", :expr_beg,
2337
+ :tSPACE, nil, :expr_beg,
2338
+ :tSTRING_END, nil, :expr_end)
2339
+ end
2340
+
2341
+ def test_yylex_string_pct_I_extra_space
2342
+ assert_lex3("%I[ s1 s2\ns3 ]",
2343
+ nil,
2344
+ :tSYMBOLS_BEG, "%I[", :expr_beg,
2345
+ :tSTRING_CONTENT, "s1", :expr_beg,
2346
+ :tSPACE, nil, :expr_beg,
2347
+ :tSTRING_CONTENT, "s2", :expr_beg,
2348
+ :tSPACE, nil, :expr_beg,
2349
+ :tSTRING_CONTENT, "s3", :expr_beg,
2350
+ :tSPACE, nil, :expr_beg,
2351
+ :tSTRING_END, nil, :expr_end)
2352
+ end
2353
+
2354
+ def test_yylex_string_pct_q
2355
+ assert_lex3("%q[s1 s2]",
2356
+ nil,
2357
+ :tSTRING_BEG, "%q[", :expr_beg,
2358
+ :tSTRING_CONTENT, "s1 s2", :expr_beg,
2359
+ :tSTRING_END, "]", :expr_end)
2160
2360
  end
2161
2361
 
2162
2362
  def test_yylex_string_pct_Q
2163
- util_lex_token("%Q[s1 s2]",
2164
- :tSTRING_BEG, "%Q[",
2165
- :tSTRING_CONTENT, "s1 s2",
2166
- :tSTRING_END, "]")
2363
+ assert_lex3("%Q[s1 s2]",
2364
+ nil,
2365
+ :tSTRING_BEG, "%Q[", :expr_beg,
2366
+ :tSTRING_CONTENT, "s1 s2", :expr_beg,
2367
+ :tSTRING_END, "]", :expr_end)
2167
2368
  end
2168
2369
 
2169
2370
  def test_yylex_string_pct_W
2170
- util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
2171
- :tWORDS_BEG, "%W[",
2172
- :tSTRING_CONTENT, "s1",
2173
- :tSPACE, nil,
2174
- :tSTRING_CONTENT, "s2",
2175
- :tSPACE, nil,
2176
- :tSTRING_CONTENT, "s3",
2177
- :tSPACE, nil,
2178
- :tSTRING_END, nil)
2371
+ assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2372
+ nil,
2373
+ :tWORDS_BEG, "%W[", :expr_beg,
2374
+ :tSTRING_CONTENT, "s1", :expr_beg,
2375
+ :tSPACE, nil, :expr_beg,
2376
+ :tSTRING_CONTENT, "s2", :expr_beg,
2377
+ :tSPACE, nil, :expr_beg,
2378
+ :tSTRING_CONTENT, "s3", :expr_beg,
2379
+ :tSPACE, nil, :expr_beg,
2380
+ :tSTRING_END, nil, :expr_end)
2179
2381
  end
2180
2382
 
2181
2383
  def test_yylex_string_pct_W_bs_nl
2182
- util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
2183
- :tWORDS_BEG, "%W[",
2184
- :tSTRING_CONTENT, "s1",
2185
- :tSPACE, nil,
2186
- :tSTRING_CONTENT, "\ns2",
2187
- :tSPACE, nil,
2188
- :tSTRING_END, nil)
2384
+ assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2385
+ nil,
2386
+ :tWORDS_BEG, "%W[", :expr_beg,
2387
+ :tSTRING_CONTENT, "s1", :expr_beg,
2388
+ :tSPACE, nil, :expr_beg,
2389
+ :tSTRING_CONTENT, "\ns2", :expr_beg,
2390
+ :tSPACE, nil, :expr_beg,
2391
+ :tSTRING_END, nil, :expr_end)
2189
2392
  end
2190
2393
 
2191
2394
  def test_yylex_string_pct_angle
2192
- util_lex_token("%<blah>",
2193
- :tSTRING_BEG, "%>",
2194
- :tSTRING_CONTENT, "blah",
2195
- :tSTRING_END, ">")
2395
+ assert_lex3("%<blah>",
2396
+ nil,
2397
+ :tSTRING_BEG, "%>", :expr_beg,
2398
+ :tSTRING_CONTENT, "blah", :expr_beg,
2399
+ :tSTRING_END, ">", :expr_end)
2196
2400
  end
2197
2401
 
2198
2402
  def test_yylex_string_pct_other
2199
- util_lex_token("%%blah%",
2200
- :tSTRING_BEG, "%%",
2201
- :tSTRING_CONTENT, "blah",
2202
- :tSTRING_END, "%")
2403
+ assert_lex3("%%blah%",
2404
+ nil,
2405
+ :tSTRING_BEG, "%%", :expr_beg,
2406
+ :tSTRING_CONTENT, "blah", :expr_beg,
2407
+ :tSTRING_END, "%", :expr_end)
2203
2408
  end
2204
2409
 
2205
2410
  def test_yylex_string_pct_w
2206
- util_bad_token("%w[s1 s2 ",
2207
- :tQWORDS_BEG, "%w[",
2208
- :tSTRING_CONTENT, "s1",
2209
- :tSPACE, nil,
2210
- :tSTRING_CONTENT, "s2",
2211
- :tSPACE, nil)
2411
+ refute_lex("%w[s1 s2 ",
2412
+ :tQWORDS_BEG, "%w[",
2413
+ :tSTRING_CONTENT, "s1",
2414
+ :tSPACE, nil,
2415
+ :tSTRING_CONTENT, "s2",
2416
+ :tSPACE, nil)
2212
2417
  end
2213
2418
 
2214
2419
  def test_yylex_string_pct_w_bs_nl
2215
- util_lex_token("%w[s1 \\\ns2]",
2216
- :tQWORDS_BEG, "%w[",
2217
- :tSTRING_CONTENT, "s1",
2218
- :tSPACE, nil,
2219
- :tSTRING_CONTENT, "\ns2",
2220
- :tSPACE, nil,
2221
- :tSTRING_END, nil)
2420
+ assert_lex3("%w[s1 \\\ns2]",
2421
+ nil,
2422
+ :tQWORDS_BEG, "%w[", :expr_beg,
2423
+ :tSTRING_CONTENT, "s1", :expr_beg,
2424
+ :tSPACE, nil, :expr_beg,
2425
+ :tSTRING_CONTENT, "\ns2", :expr_beg,
2426
+ :tSPACE, nil, :expr_beg,
2427
+ :tSTRING_END, nil, :expr_end)
2222
2428
  end
2223
2429
 
2224
2430
  def test_yylex_string_pct_w_bs_sp
2225
- util_lex_token("%w[s\\ 1 s\\ 2]",
2226
- :tQWORDS_BEG, "%w[",
2227
- :tSTRING_CONTENT, "s 1",
2228
- :tSPACE, nil,
2229
- :tSTRING_CONTENT, "s 2",
2230
- :tSPACE, nil,
2231
- :tSTRING_END, nil)
2431
+ assert_lex3("%w[s\\ 1 s\\ 2]",
2432
+ nil,
2433
+ :tQWORDS_BEG, "%w[", :expr_beg,
2434
+ :tSTRING_CONTENT, "s 1", :expr_beg,
2435
+ :tSPACE, nil, :expr_beg,
2436
+ :tSTRING_CONTENT, "s 2", :expr_beg,
2437
+ :tSPACE, nil, :expr_beg,
2438
+ :tSTRING_END, nil, :expr_end)
2232
2439
  end
2233
2440
 
2234
2441
  def test_yylex_string_single
2235
- util_lex_token("'string'",
2236
- :tSTRING, "string")
2442
+ assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
2237
2443
  end
2238
2444
 
2239
2445
  def test_yylex_string_single_escape_chars
2240
- util_lex_token("'s\\tri\\ng'",
2241
- :tSTRING, "s\\tri\\ng")
2446
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
2242
2447
  end
2243
2448
 
2244
2449
  def test_yylex_string_single_nl
2245
- util_lex_token("'blah\\\nblah'",
2246
- :tSTRING, "blah\\\nblah")
2450
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
2247
2451
  end
2248
2452
 
2249
2453
  def test_yylex_symbol
2250
- util_lex_token(":symbol",
2251
- :tSYMBOL, "symbol")
2454
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
2252
2455
  end
2253
2456
 
2254
- def test_yylex_symbol_bad_zero
2255
- util_bad_token(":\"blah\0\"",
2256
- :tSYMBEG, ":")
2457
+ def test_yylex_symbol_zero_byte__18
2458
+ setup_lexer_class Ruby18Parser
2459
+
2460
+ refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
2461
+ end
2462
+
2463
+ def test_yylex_symbol_zero_byte
2464
+ assert_lex(":\"symbol\0\"", nil,
2465
+ :tSYMBOL, "symbol\0", :expr_end)
2257
2466
  end
2258
2467
 
2259
2468
  def test_yylex_symbol_double
2260
- util_lex_token(":\"symbol\"",
2261
- :tSYMBEG, ":",
2262
- :tSTRING_CONTENT, "symbol",
2263
- :tSTRING_END, '"')
2469
+ assert_lex3(":\"symbol\"",
2470
+ nil,
2471
+ :tSYMBOL, "symbol", :expr_end)
2472
+ end
2473
+
2474
+ def test_yylex_symbol_double_interp
2475
+ assert_lex3(':"symbol#{1+1}"',
2476
+ nil,
2477
+ :tSYMBEG, ":", :expr_fname,
2478
+ :tSTRING_CONTENT, "symbol", :expr_fname,
2479
+ :tSTRING_DBEG, nil, :expr_fname,
2480
+ :tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
2481
+ :tSTRING_END, "\"", :expr_end)
2264
2482
  end
2265
2483
 
2266
2484
  def test_yylex_symbol_single
2267
- util_lex_token(":'symbol'",
2268
- :tSYMBEG, ":",
2269
- :tSTRING_CONTENT, "symbol",
2270
- :tSTRING_END, "'")
2485
+ assert_lex3(":'symbol'",
2486
+ nil,
2487
+ :tSYMBOL, "symbol", :expr_end)
2271
2488
  end
2272
2489
 
2273
- def test_yylex_ternary
2274
- util_lex_token("a ? b : c",
2275
- :tIDENTIFIER, "a",
2276
- :tEH, "?",
2277
- :tIDENTIFIER, "b",
2278
- :tCOLON, ":",
2279
- :tIDENTIFIER, "c")
2490
+ def test_yylex_symbol_single_noninterp
2491
+ assert_lex3(':\'symbol#{1+1}\'',
2492
+ nil,
2493
+ :tSYMBOL, 'symbol#{1+1}', :expr_end)
2494
+ end
2280
2495
 
2281
- util_lex_token("a ?bb : c", # GAH! MATZ!!!
2282
- :tIDENTIFIER, "a",
2283
- :tEH, "?",
2284
- :tIDENTIFIER, "bb",
2285
- :tCOLON, ":",
2286
- :tIDENTIFIER, "c")
2496
+ def test_yylex_ternary1
2497
+ assert_lex3("a ? b : c",
2498
+ nil,
2499
+ :tIDENTIFIER, "a", :expr_cmdarg,
2500
+ :tEH, "?", :expr_value,
2501
+ :tIDENTIFIER, "b", :expr_arg,
2502
+ :tCOLON, ":", :expr_beg,
2503
+ :tIDENTIFIER, "c", :expr_arg)
2504
+
2505
+ assert_lex3("a ?bb : c", # GAH! MATZ!!!
2506
+ nil,
2507
+ :tIDENTIFIER, "a", :expr_cmdarg,
2508
+ :tEH, "?", :expr_beg,
2509
+ :tIDENTIFIER, "bb", :expr_arg,
2510
+ :tCOLON, ":", :expr_beg,
2511
+ :tIDENTIFIER, "c", :expr_arg)
2287
2512
 
2288
- util_lex_token("42 ?", # 42 forces expr_end
2289
- :tINTEGER, 42,
2290
- :tEH, "?")
2513
+ assert_lex3("42 ?",
2514
+ nil,
2515
+ :tINTEGER, 42, :expr_end,
2516
+ :tEH, "?", :expr_value)
2291
2517
  end
2292
2518
 
2293
2519
  def test_yylex_tilde
2294
- util_lex_token "~", :tTILDE, "~"
2520
+ assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
2295
2521
  end
2296
2522
 
2297
2523
  def test_yylex_tilde_unary
2298
- @lex.lex_state = :expr_fname
2299
- util_lex_token "~@", :tTILDE, "~"
2524
+ self.lex_state = :expr_fname
2525
+
2526
+ assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
2300
2527
  end
2301
2528
 
2302
2529
  def test_yylex_uminus
2303
- util_lex_token("-blah",
2304
- :tUMINUS, "-",
2305
- :tIDENTIFIER, "blah")
2530
+ assert_lex3("-blah",
2531
+ nil,
2532
+ :tUMINUS, "-", :expr_beg,
2533
+ :tIDENTIFIER, "blah", :expr_arg)
2306
2534
  end
2307
2535
 
2308
2536
  def test_yylex_underscore
2309
- util_lex_token("_var", :tIDENTIFIER, "_var")
2537
+ assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
2310
2538
  end
2311
2539
 
2312
2540
  def test_yylex_underscore_end
2313
2541
  @lex.src = "__END__\n"
2314
- refute @lex.advance
2542
+ refute_lexeme
2315
2543
  end
2316
2544
 
2317
2545
  def test_yylex_uplus
2318
- util_lex_token("+blah",
2319
- :tUPLUS, "+",
2320
- :tIDENTIFIER, "blah")
2546
+ assert_lex3("+blah",
2547
+ nil,
2548
+ :tUPLUS, "+", :expr_beg,
2549
+ :tIDENTIFIER, "blah", :expr_arg)
2321
2550
  end
2322
2551
 
2323
2552
  def test_zbug_float_in_decl
2324
- util_lex_token("def initialize(u = ",
2325
- :kDEF, "def",
2326
- :tIDENTIFIER, "initialize",
2327
- :tLPAREN2, "(",
2328
- :tIDENTIFIER, "u",
2329
- :tEQL, "=")
2330
-
2331
- assert_equal :expr_beg, @lex.lex_state
2332
-
2333
- util_lex_token("0.0, s = 0.0",
2334
- :tFLOAT, 0.0,
2335
- :tCOMMA, ',',
2336
- :tIDENTIFIER, "s",
2337
- :tEQL, "=",
2338
- :tFLOAT, 0.0)
2553
+ assert_lex3("def initialize(u = 0.0, s = 0.0",
2554
+ nil,
2555
+ :kDEF, "def", :expr_fname,
2556
+ :tIDENTIFIER, "initialize", :expr_endfn,
2557
+ :tLPAREN2, "(", :expr_beg,
2558
+ :tIDENTIFIER, "u", :expr_arg,
2559
+ :tEQL, "=", :expr_beg,
2560
+ :tFLOAT, 0.0, :expr_end,
2561
+ :tCOMMA, ",", :expr_beg,
2562
+ :tIDENTIFIER, "s", :expr_arg,
2563
+ :tEQL, "=", :expr_beg,
2564
+ :tFLOAT, 0.0, :expr_end)
2339
2565
  end
2340
2566
 
2341
2567
  def test_zbug_id_equals
2342
- util_lex_token("a =",
2343
- :tIDENTIFIER, "a",
2344
- :tEQL, "=")
2345
-
2346
- assert_equal :expr_beg, @lex.lex_state
2347
-
2348
- util_lex_token("0.0",
2349
- :tFLOAT, 0.0)
2568
+ assert_lex3("a = 0.0",
2569
+ nil,
2570
+ :tIDENTIFIER, "a", :expr_cmdarg,
2571
+ :tEQL, "=", :expr_beg,
2572
+ :tFLOAT, 0.0, :expr_end)
2350
2573
  end
2351
2574
 
2352
2575
  def test_zbug_no_spaces_in_decl
2353
- util_lex_token("def initialize(u=",
2354
- :kDEF, "def",
2355
- :tIDENTIFIER, "initialize",
2356
- :tLPAREN2, "(",
2357
- :tIDENTIFIER, "u",
2358
- :tEQL, "=")
2359
-
2360
- assert_equal :expr_beg, @lex.lex_state
2361
-
2362
- util_lex_token("0.0,s=0.0",
2363
- :tFLOAT, 0.0,
2364
- :tCOMMA, ",",
2365
- :tIDENTIFIER, "s",
2366
- :tEQL, "=",
2367
- :tFLOAT, 0.0)
2576
+ assert_lex3("def initialize(u=0.0,s=0.0",
2577
+ nil,
2578
+ :kDEF, "def", :expr_fname,
2579
+ :tIDENTIFIER, "initialize", :expr_endfn,
2580
+ :tLPAREN2, "(", :expr_beg,
2581
+ :tIDENTIFIER, "u", :expr_arg,
2582
+ :tEQL, "=", :expr_beg,
2583
+ :tFLOAT, 0.0, :expr_end,
2584
+ :tCOMMA, ",", :expr_beg,
2585
+ :tIDENTIFIER, "s", :expr_arg,
2586
+ :tEQL, "=", :expr_beg,
2587
+ :tFLOAT, 0.0, :expr_end)
2368
2588
  end
2369
2589
 
2370
2590
  def test_pct_w_backslashes
@@ -2381,53 +2601,5 @@ class TestRubyLexer < Minitest::Test
2381
2601
  :tSPACE, nil, :expr_beg, 0, 0,
2382
2602
  :tSTRING_END, nil, :expr_end, 0, 0)
2383
2603
  end
2384
-
2385
- # flunk "Not yet"
2386
- end
2387
-
2388
- ############################################################
2389
-
2390
- def util_bad_token s, *args
2391
- assert_raises RubyParser::SyntaxError do
2392
- util_lex_token s, *args
2393
- end
2394
- end
2395
-
2396
- def util_escape expected, input
2397
- @lex.src = input
2398
- assert_equal expected, @lex.read_escape, input
2399
- end
2400
-
2401
- def util_escape_bad input
2402
- @lex.src = input
2403
- assert_raises RubyParser::SyntaxError do
2404
- @lex.read_escape
2405
- end
2406
- end
2407
-
2408
- def util_lex_fname name, type, end_state = :expr_arg
2409
- @lex.lex_state = :expr_fname # can only set via parser's defs
2410
-
2411
- assert_lex("def #{name} ",
2412
- nil,
2413
-
2414
- :kDEF, "def", :expr_fname, 0, 0,
2415
- type, name, end_state, 0, 0)
2416
-
2417
- assert_equal end_state, @lex.lex_state
2418
- end
2419
-
2420
- def util_lex_token input, *args
2421
- @lex.src = input
2422
-
2423
- until args.empty? do
2424
- token = args.shift
2425
- value = args.shift
2426
- assert @lex.advance, "no more tokens"
2427
- # assert_equal [token, value].map(&:encoding), [@lex.token, [@lex.yacc_value].flatten.first].map(&:encoding), input # TODO
2428
- assert_equal [token, value], [@lex.token, [@lex.yacc_value].flatten.first], input
2429
- end
2430
-
2431
- refute @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
2432
2604
  end
2433
2605
  end