ruby_parser 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ruby_parser might be problematic. Click here for more details.
- data/.autotest +26 -3
- data/History.txt +108 -0
- data/Manifest.txt +3 -0
- data/README.txt +1 -1
- data/Rakefile +126 -28
- data/bin/ruby_parse +89 -0
- data/lib/ruby_lexer.rb +1117 -2536
- data/lib/ruby_parser.rb +5407 -5849
- data/lib/ruby_parser.y +1763 -1621
- data/lib/ruby_parser_extras.rb +1051 -0
- data/test/test_ruby_lexer.rb +1607 -267
- data/test/test_ruby_parser.rb +317 -175
- data/test/test_ruby_parser_extras.rb +177 -0
- metadata +27 -10
data/test/test_ruby_lexer.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/local/bin/ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "test/unit"
|
4
|
+
require "ruby_lexer"
|
5
5
|
|
6
6
|
class TestRubyLexer < Test::Unit::TestCase
|
7
7
|
def deny cond, msg = nil
|
@@ -9,9 +9,10 @@ class TestRubyLexer < Test::Unit::TestCase
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def setup
|
12
|
-
|
13
|
-
@lex
|
14
|
-
@lex.
|
12
|
+
p = RubyParser.new
|
13
|
+
@lex = p.lexer
|
14
|
+
@lex.src = "blah blah"
|
15
|
+
@lex.lex_state = :expr_beg
|
15
16
|
end
|
16
17
|
|
17
18
|
def test_advance
|
@@ -20,379 +21,1718 @@ class TestRubyLexer < Test::Unit::TestCase
|
|
20
21
|
deny @lex.advance # nada
|
21
22
|
end
|
22
23
|
|
23
|
-
def
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
def test_read_escape
|
25
|
+
util_escape "\\", "\\"
|
26
|
+
util_escape "\n", "n"
|
27
|
+
util_escape "\t", "t"
|
28
|
+
util_escape "\r", "r"
|
29
|
+
util_escape "\f", "f"
|
30
|
+
util_escape "\13", "v"
|
31
|
+
util_escape "\0", "0"
|
32
|
+
util_escape "\07", "a"
|
33
|
+
util_escape "\007", "a"
|
34
|
+
util_escape "\033", "e"
|
35
|
+
util_escape "\377", "377"
|
36
|
+
util_escape "\377", "xff"
|
37
|
+
util_escape "\010", "b"
|
38
|
+
util_escape " ", "s"
|
39
|
+
util_escape "q", "q" # plain vanilla escape
|
29
40
|
end
|
30
41
|
|
31
|
-
def
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
42
|
+
def test_read_escape_c
|
43
|
+
util_escape "\030", "C-x"
|
44
|
+
util_escape "\030", "cx"
|
45
|
+
util_escape "\230", 'C-\M-x'
|
46
|
+
util_escape "\230", 'c\M-x'
|
47
|
+
|
48
|
+
util_escape "\177", "C-?"
|
49
|
+
util_escape "\177", "c?"
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_read_escape_errors
|
53
|
+
util_escape_bad ""
|
54
|
+
|
55
|
+
util_escape_bad "M"
|
56
|
+
util_escape_bad "M-"
|
57
|
+
util_escape_bad "Mx"
|
58
|
+
|
59
|
+
util_escape_bad "Cx"
|
60
|
+
util_escape_bad "C"
|
61
|
+
util_escape_bad "C-"
|
62
|
+
|
63
|
+
util_escape_bad "c"
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_read_escape_m
|
67
|
+
util_escape "\370", "M-x"
|
68
|
+
util_escape "\230", 'M-\C-x'
|
69
|
+
util_escape "\230", 'M-\cx'
|
70
|
+
end
|
71
|
+
|
72
|
+
def test_yylex_ambiguous_uminus
|
73
|
+
util_lex_token("m -3",
|
74
|
+
:tIDENTIFIER, "m",
|
75
|
+
:tUMINUS_NUM, "-",
|
76
|
+
:tINTEGER, 3)
|
77
|
+
# TODO: verify warning
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_yylex_ambiguous_uplus
|
81
|
+
util_lex_token("m +3",
|
82
|
+
:tIDENTIFIER, "m",
|
83
|
+
:tINTEGER, 3)
|
84
|
+
# TODO: verify warning
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_yylex_and
|
88
|
+
util_lex_token "&", :tAMPER, "&"
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_yylex_and2
|
92
|
+
util_lex_token "&&", :tANDOP, "&&"
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_yylex_and2_equals
|
96
|
+
util_lex_token "&&=", :tOP_ASGN, "&&"
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_yylex_and_arg
|
100
|
+
@lex.lex_state = :expr_arg
|
101
|
+
|
102
|
+
util_lex_token(" &y",
|
103
|
+
:tAMPER, "&",
|
104
|
+
:tIDENTIFIER, "y")
|
105
|
+
end
|
106
|
+
|
107
|
+
def test_yylex_and_equals
|
108
|
+
util_lex_token "&=", :tOP_ASGN, "&"
|
109
|
+
end
|
110
|
+
|
111
|
+
def test_yylex_and_expr
|
112
|
+
@lex.lex_state = :expr_arg
|
113
|
+
|
114
|
+
util_lex_token("x & y",
|
115
|
+
:tIDENTIFIER, "x",
|
116
|
+
:tAMPER2, "&",
|
117
|
+
:tIDENTIFIER, "y")
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_yylex_and_meth
|
121
|
+
util_lex_fname "&", :tAMPER2
|
122
|
+
end
|
123
|
+
|
124
|
+
def test_yylex_assoc
|
125
|
+
util_lex_token "=>", :tASSOC, "=>"
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_yylex_back_ref
|
129
|
+
util_lex_token("[$&, $`, $', $+]",
|
130
|
+
:tLBRACK, "[",
|
131
|
+
:tBACK_REF, :"&", :tCOMMA, ",",
|
132
|
+
:tBACK_REF, :"`", :tCOMMA, ",",
|
133
|
+
:tBACK_REF, :"'", :tCOMMA, ",",
|
134
|
+
:tBACK_REF, :"+",
|
135
|
+
:tRBRACK, "]")
|
136
|
+
end
|
137
|
+
|
138
|
+
def test_yylex_backslash
|
139
|
+
util_lex_token("1 \\\n+ 2",
|
140
|
+
:tINTEGER, 1,
|
141
|
+
:tPLUS, "+",
|
142
|
+
:tINTEGER, 2)
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_yylex_backslash_bad
|
146
|
+
util_bad_token("1 \\ + 2",
|
147
|
+
:tINTEGER, 1)
|
148
|
+
end
|
149
|
+
|
150
|
+
def test_yylex_backtick
|
151
|
+
util_lex_token("`ls`",
|
152
|
+
:tXSTRING_BEG, "`",
|
153
|
+
:tSTRING_CONTENT, "ls",
|
154
|
+
:tSTRING_END, "`")
|
155
|
+
end
|
156
|
+
|
157
|
+
def test_yylex_backtick_cmdarg
|
158
|
+
@lex.lex_state = :expr_dot
|
159
|
+
util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
|
160
|
+
|
161
|
+
assert_equal :expr_cmdarg, @lex.lex_state
|
162
|
+
end
|
163
|
+
|
164
|
+
def test_yylex_backtick_dot
|
165
|
+
@lex.lex_state = :expr_dot
|
166
|
+
util_lex_token("a.`(3)",
|
167
|
+
:tIDENTIFIER, "a",
|
168
|
+
:tDOT, ".",
|
169
|
+
:tBACK_REF2, "`",
|
170
|
+
:tLPAREN2, "(",
|
171
|
+
:tINTEGER, 3,
|
172
|
+
:tRPAREN, ")")
|
173
|
+
end
|
174
|
+
|
175
|
+
def test_yylex_backtick_method
|
176
|
+
@lex.lex_state = :expr_fname
|
177
|
+
util_lex_token("`", :tBACK_REF2, "`")
|
178
|
+
assert_equal :expr_end, @lex.lex_state
|
179
|
+
end
|
180
|
+
|
181
|
+
def test_yylex_bad_char
|
182
|
+
util_bad_token(" \010 ")
|
183
|
+
end
|
184
|
+
|
185
|
+
def test_yylex_bang
|
186
|
+
util_lex_token "!", :tBANG, "!"
|
187
|
+
end
|
188
|
+
|
189
|
+
def test_yylex_bang_equals
|
190
|
+
util_lex_token "!=", :tNEQ, "!="
|
191
|
+
end
|
192
|
+
|
193
|
+
def test_yylex_bang_tilde
|
194
|
+
util_lex_token "!~", :tNMATCH, "!~"
|
195
|
+
end
|
196
|
+
|
197
|
+
def test_yylex_carat
|
198
|
+
util_lex_token "^", :tCARET, "^"
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_yylex_carat_equals
|
202
|
+
util_lex_token "^=", :tOP_ASGN, "^"
|
203
|
+
end
|
204
|
+
|
205
|
+
def test_yylex_colon2
|
206
|
+
util_lex_token("A::B",
|
207
|
+
:tCONSTANT, "A",
|
208
|
+
:tCOLON2, "::",
|
209
|
+
:tCONSTANT, "B")
|
210
|
+
end
|
211
|
+
|
212
|
+
def test_yylex_colon3
|
213
|
+
util_lex_token("::Array",
|
214
|
+
:tCOLON3, "::",
|
215
|
+
:tCONSTANT, "Array")
|
216
|
+
end
|
217
|
+
|
218
|
+
def test_yylex_comma
|
219
|
+
util_lex_token ",", :tCOMMA, ","
|
220
|
+
end
|
221
|
+
|
222
|
+
def test_yylex_comment
|
223
|
+
util_lex_token("1 # one\n# two\n2",
|
224
|
+
:tINTEGER, 1,
|
225
|
+
:tNL, nil,
|
226
|
+
:tINTEGER, 2)
|
227
|
+
assert_equal "# one\n# two\n", @lex.comments
|
228
|
+
end
|
229
|
+
|
230
|
+
def test_yylex_comment_begin
|
231
|
+
util_lex_token("=begin\nblah\nblah\n=end\n42",
|
232
|
+
:tINTEGER, 42)
|
233
|
+
assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
|
234
|
+
end
|
235
|
+
|
236
|
+
def test_yylex_comment_begin_bad
|
237
|
+
util_bad_token("=begin\nblah\nblah\n")
|
238
|
+
assert_equal "", @lex.comments
|
239
|
+
end
|
240
|
+
|
241
|
+
def test_yylex_comment_begin_not_comment
|
242
|
+
util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
|
243
|
+
:tIDENTIFIER, "beginfoo",
|
244
|
+
:tEQL, "=",
|
245
|
+
:tINTEGER, 5,
|
246
|
+
:tNL, nil,
|
247
|
+
:tIDENTIFIER, "p",
|
248
|
+
:tIDENTIFIER, "x",
|
249
|
+
:tEQL, "=",
|
250
|
+
:tIDENTIFIER, "beginfoo")
|
251
|
+
end
|
252
|
+
|
253
|
+
def test_yylex_comment_begin_space
|
254
|
+
util_lex_token("=begin blah\nblah\n=end\n")
|
255
|
+
assert_equal "=begin blah\nblah\n=end\n", @lex.comments
|
256
|
+
end
|
257
|
+
|
258
|
+
def test_yylex_comment_eos
|
259
|
+
util_lex_token("# comment")
|
260
|
+
end
|
261
|
+
|
262
|
+
def test_yylex_constant
|
263
|
+
util_lex_token("ArgumentError",
|
264
|
+
:tCONSTANT, "ArgumentError")
|
265
|
+
end
|
266
|
+
|
267
|
+
def test_yylex_constant_semi
|
268
|
+
util_lex_token("ArgumentError;",
|
269
|
+
:tCONSTANT, "ArgumentError",
|
270
|
+
:tSEMI, ";")
|
271
|
+
end
|
272
|
+
|
273
|
+
def test_yylex_cvar
|
274
|
+
util_lex_token "@@blah", :tCVAR, "@@blah"
|
275
|
+
end
|
276
|
+
|
277
|
+
def test_yylex_cvar_bad
|
278
|
+
assert_raises SyntaxError do
|
279
|
+
util_lex_token "@@1"
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
def test_yylex_def_bad_name
|
284
|
+
@lex.lex_state = :expr_fname
|
285
|
+
util_bad_token("def [ ", :kDEF, "def")
|
286
|
+
end
|
287
|
+
|
288
|
+
def test_yylex_div
|
289
|
+
util_lex_token("a / 2",
|
290
|
+
:tIDENTIFIER, "a",
|
291
|
+
:tDIVIDE, "/",
|
292
|
+
:tINTEGER, 2)
|
293
|
+
end
|
294
|
+
|
295
|
+
def test_yylex_div_equals
|
296
|
+
util_lex_token("a /= 2",
|
297
|
+
:tIDENTIFIER, "a",
|
298
|
+
:tOP_ASGN, "/",
|
299
|
+
:tINTEGER, 2)
|
300
|
+
end
|
301
|
+
|
302
|
+
def test_yylex_do
|
303
|
+
util_lex_token("x do 42 end",
|
304
|
+
:tIDENTIFIER, "x",
|
305
|
+
:kDO, "do",
|
306
|
+
:tINTEGER, 42,
|
307
|
+
:kEND, "end")
|
308
|
+
end
|
309
|
+
|
310
|
+
def test_yylex_do_block
|
311
|
+
@lex.lex_state = :expr_endarg
|
312
|
+
@lex.cmdarg.push true
|
313
|
+
|
314
|
+
util_lex_token("x.y do 42 end",
|
315
|
+
:tIDENTIFIER, "x",
|
316
|
+
:tDOT, ".",
|
317
|
+
:tIDENTIFIER, "y",
|
318
|
+
:kDO_BLOCK, "do",
|
319
|
+
:tINTEGER, 42,
|
320
|
+
:kEND, "end")
|
321
|
+
end
|
322
|
+
|
323
|
+
def test_yylex_do_block2
|
324
|
+
@lex.lex_state = :expr_endarg
|
325
|
+
|
326
|
+
util_lex_token("do 42 end",
|
327
|
+
:kDO_BLOCK, "do",
|
328
|
+
:tINTEGER, 42,
|
329
|
+
:kEND, "end")
|
330
|
+
end
|
331
|
+
|
332
|
+
def test_yylex_do_cond
|
333
|
+
@lex.cond.push true
|
334
|
+
|
335
|
+
util_lex_token("x do 42 end",
|
336
|
+
:tIDENTIFIER, "x",
|
337
|
+
:kDO_COND, "do",
|
338
|
+
:tINTEGER, 42,
|
339
|
+
:kEND, "end")
|
340
|
+
end
|
341
|
+
|
342
|
+
def test_yylex_dollar
|
343
|
+
util_lex_token("$", "$", "$") # FIX: wtf is this?!?
|
344
|
+
end
|
345
|
+
|
346
|
+
def test_yylex_dot # HINT message sends
|
347
|
+
util_lex_token ".", :tDOT, "."
|
348
|
+
end
|
349
|
+
|
350
|
+
def test_yylex_dot2
|
351
|
+
util_lex_token "..", :tDOT2, ".."
|
352
|
+
end
|
353
|
+
|
354
|
+
def test_yylex_dot3
|
355
|
+
util_lex_token "...", :tDOT3, "..."
|
356
|
+
end
|
357
|
+
|
358
|
+
def test_yylex_equals
|
359
|
+
util_lex_token "=", :tEQL, "=" # FIX: this sucks
|
360
|
+
end
|
361
|
+
|
362
|
+
def test_yylex_equals2
|
363
|
+
util_lex_token "==", :tEQ, "=="
|
364
|
+
end
|
365
|
+
|
366
|
+
def test_yylex_equals3
|
367
|
+
util_lex_token "===", :tEQQ, "==="
|
368
|
+
end
|
369
|
+
|
370
|
+
def test_yylex_equals_tilde
|
371
|
+
util_lex_token "=~", :tMATCH, "=~"
|
372
|
+
end
|
373
|
+
|
374
|
+
def test_yylex_float
|
375
|
+
util_lex_token "1.0", :tFLOAT, 1.0
|
376
|
+
end
|
377
|
+
|
378
|
+
def test_yylex_float_bad_no_underscores
|
379
|
+
util_bad_token "1__0.0"
|
380
|
+
end
|
381
|
+
|
382
|
+
def test_yylex_float_bad_no_zero_leading
|
383
|
+
util_bad_token ".0"
|
384
|
+
end
|
385
|
+
|
386
|
+
def test_yylex_float_bad_trailing_underscore
|
387
|
+
util_bad_token "123_.0"
|
388
|
+
end
|
389
|
+
|
390
|
+
def test_yylex_float_call
|
391
|
+
util_lex_token("1.0.to_s",
|
392
|
+
:tFLOAT, 1.0,
|
393
|
+
:tDOT, ".",
|
394
|
+
:tIDENTIFIER, "to_s")
|
395
|
+
end
|
396
|
+
|
397
|
+
def test_yylex_float_dot_E
|
398
|
+
util_lex_token "1.0E10", :tFLOAT, 1.0e10
|
399
|
+
end
|
400
|
+
|
401
|
+
def test_yylex_float_dot_E_neg
|
402
|
+
util_lex_token("-1.0E10",
|
403
|
+
:tUMINUS_NUM, "-",
|
404
|
+
:tFLOAT, 1.0e10)
|
405
|
+
end
|
406
|
+
|
407
|
+
def test_yylex_float_dot_e
|
408
|
+
util_lex_token "1.0e10", :tFLOAT, 1.0e10
|
409
|
+
end
|
410
|
+
|
411
|
+
def test_yylex_float_dot_e_neg
|
412
|
+
util_lex_token("-1.0e10",
|
413
|
+
:tUMINUS_NUM, "-",
|
414
|
+
:tFLOAT, 1.0e10)
|
415
|
+
end
|
416
|
+
|
417
|
+
def test_yylex_float_e
|
418
|
+
util_lex_token "1e10", :tFLOAT, 1e10
|
419
|
+
end
|
420
|
+
|
421
|
+
def test_yylex_float_e_bad_double_e
|
422
|
+
util_bad_token "1e2e3"
|
423
|
+
end
|
424
|
+
|
425
|
+
def test_yylex_float_e_bad_trailing_underscore
|
426
|
+
util_bad_token "123_e10"
|
427
|
+
end
|
428
|
+
|
429
|
+
def test_yylex_float_e_minus
|
430
|
+
util_lex_token "1e-10", :tFLOAT, 1e-10
|
431
|
+
end
|
432
|
+
|
433
|
+
def test_yylex_float_e_neg
|
434
|
+
util_lex_token("-1e10",
|
435
|
+
:tUMINUS_NUM, "-",
|
436
|
+
:tFLOAT, 1e10)
|
437
|
+
end
|
438
|
+
|
439
|
+
def test_yylex_float_e_neg_minus
|
440
|
+
util_lex_token("-1e-10",
|
441
|
+
:tUMINUS_NUM, "-",
|
442
|
+
:tFLOAT, 1e-10)
|
443
|
+
end
|
444
|
+
|
445
|
+
def test_yylex_float_e_neg_plus
|
446
|
+
util_lex_token("-1e+10",
|
447
|
+
:tUMINUS_NUM, "-",
|
448
|
+
:tFLOAT, 1e10)
|
449
|
+
end
|
450
|
+
|
451
|
+
def test_yylex_float_e_plus
|
452
|
+
util_lex_token "1e+10", :tFLOAT, 1e10
|
453
|
+
end
|
454
|
+
|
455
|
+
def test_yylex_float_e_zero
|
456
|
+
util_lex_token "0e0", :tFLOAT, 0e0
|
457
|
+
end
|
458
|
+
|
459
|
+
def test_yylex_float_neg
|
460
|
+
util_lex_token("-1.0",
|
461
|
+
:tUMINUS_NUM, "-",
|
462
|
+
:tFLOAT, 1.0)
|
463
|
+
end
|
464
|
+
|
465
|
+
def test_yylex_ge
|
466
|
+
util_lex_token("a >= 2",
|
467
|
+
:tIDENTIFIER, "a",
|
468
|
+
:tGEQ, ">=",
|
469
|
+
:tINTEGER, 2)
|
470
|
+
end
|
471
|
+
|
472
|
+
def test_yylex_global
|
473
|
+
util_lex_token("$blah", :tGVAR, "$blah")
|
474
|
+
end
|
475
|
+
|
476
|
+
def test_yylex_global_backref
|
477
|
+
@lex.lex_state = :expr_fname
|
478
|
+
util_lex_token("$`", :tGVAR, "$`")
|
479
|
+
end
|
480
|
+
|
481
|
+
def test_yylex_global_dash_nothing
|
482
|
+
util_lex_token("$- ", :tGVAR, "$-")
|
483
|
+
end
|
484
|
+
|
485
|
+
def test_yylex_global_dash_something
|
486
|
+
util_lex_token("$-x", :tGVAR, "$-x")
|
487
|
+
end
|
488
|
+
|
489
|
+
def test_yylex_global_number
|
490
|
+
@lex.lex_state = :expr_fname
|
491
|
+
util_lex_token("$1", :tGVAR, "$1")
|
492
|
+
end
|
493
|
+
|
494
|
+
def test_yylex_global_number_big
|
495
|
+
@lex.lex_state = :expr_fname
|
496
|
+
util_lex_token("$1234", :tGVAR, "$1234")
|
497
|
+
end
|
498
|
+
|
499
|
+
def test_yylex_global_other
|
500
|
+
util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
|
501
|
+
:tLBRACK, "[",
|
502
|
+
:tGVAR, "$~", :tCOMMA, ",",
|
503
|
+
:tGVAR, "$*", :tCOMMA, ",",
|
504
|
+
:tGVAR, "$$", :tCOMMA, ",",
|
505
|
+
:tGVAR, "$\?", :tCOMMA, ",",
|
506
|
+
:tGVAR, "$!", :tCOMMA, ",",
|
507
|
+
:tGVAR, "$@", :tCOMMA, ",",
|
508
|
+
:tGVAR, "$/", :tCOMMA, ",",
|
509
|
+
:tGVAR, "$\\", :tCOMMA, ",",
|
510
|
+
:tGVAR, "$;", :tCOMMA, ",",
|
511
|
+
:tGVAR, "$,", :tCOMMA, ",",
|
512
|
+
:tGVAR, "$.", :tCOMMA, ",",
|
513
|
+
:tGVAR, "$=", :tCOMMA, ",",
|
514
|
+
:tGVAR, "$:", :tCOMMA, ",",
|
515
|
+
:tGVAR, "$<", :tCOMMA, ",",
|
516
|
+
:tGVAR, "$>", :tCOMMA, ",",
|
517
|
+
:tGVAR, "$\"",
|
518
|
+
:tRBRACK, "]")
|
519
|
+
end
|
520
|
+
|
521
|
+
def test_yylex_global_underscore
|
522
|
+
util_lex_token("$_",
|
523
|
+
:tGVAR, "$_")
|
524
|
+
end
|
525
|
+
|
526
|
+
def test_yylex_global_wierd
|
527
|
+
util_lex_token("$__blah",
|
528
|
+
:tGVAR, "$__blah")
|
529
|
+
end
|
530
|
+
|
531
|
+
def test_yylex_global_zero
|
532
|
+
util_lex_token("$0", :tGVAR, "$0")
|
533
|
+
end
|
534
|
+
|
535
|
+
def test_yylex_gt
|
536
|
+
util_lex_token("a > 2",
|
537
|
+
:tIDENTIFIER, "a",
|
538
|
+
:tGT, ">",
|
539
|
+
:tINTEGER, 2)
|
540
|
+
end
|
541
|
+
|
542
|
+
def test_yylex_heredoc_backtick
|
543
|
+
util_lex_token("a = <<`EOF`\n blah blah\nEOF\n",
|
544
|
+
:tIDENTIFIER, "a",
|
545
|
+
:tEQL, "=",
|
546
|
+
:tXSTRING_BEG, "`",
|
547
|
+
:tSTRING_CONTENT, " blah blah\n",
|
548
|
+
:tSTRING_END, "EOF",
|
549
|
+
:tNL, nil)
|
550
|
+
end
|
551
|
+
|
552
|
+
def test_yylex_heredoc_double
|
553
|
+
util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n",
|
554
|
+
:tIDENTIFIER, "a",
|
555
|
+
:tEQL, "=",
|
556
|
+
:tSTRING_BEG, "\"",
|
557
|
+
:tSTRING_CONTENT, " blah blah\n",
|
558
|
+
:tSTRING_END, "EOF",
|
559
|
+
:tNL, nil)
|
560
|
+
end
|
561
|
+
|
562
|
+
def test_yylex_heredoc_double_dash
|
563
|
+
util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n",
|
564
|
+
:tIDENTIFIER, "a",
|
565
|
+
:tEQL, "=",
|
566
|
+
:tSTRING_BEG, "\"",
|
567
|
+
:tSTRING_CONTENT, " blah blah\n",
|
568
|
+
:tSTRING_END, "EOF",
|
569
|
+
:tNL, nil)
|
570
|
+
end
|
571
|
+
|
572
|
+
def test_yylex_heredoc_double_eos
|
573
|
+
util_bad_token("a = <<\"EOF\"\nblah",
|
574
|
+
:tIDENTIFIER, "a",
|
575
|
+
:tEQL, "=",
|
576
|
+
:tSTRING_BEG, "\"")
|
577
|
+
end
|
578
|
+
|
579
|
+
def test_yylex_heredoc_double_eos_nl
|
580
|
+
util_bad_token("a = <<\"EOF\"\nblah\n",
|
581
|
+
:tIDENTIFIER, "a",
|
582
|
+
:tEQL, "=",
|
583
|
+
:tSTRING_BEG, "\"")
|
584
|
+
end
|
585
|
+
|
586
|
+
def test_yylex_heredoc_double_interp
|
587
|
+
util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
|
588
|
+
:tIDENTIFIER, "a",
|
589
|
+
:tEQL, "=",
|
590
|
+
:tSTRING_BEG, "\"",
|
591
|
+
:tSTRING_CONTENT, "#x a ",
|
592
|
+
:tSTRING_DVAR, "\#@",
|
593
|
+
:tSTRING_CONTENT, "@a b ", # HUH?
|
594
|
+
:tSTRING_DVAR, "\#$",
|
595
|
+
:tSTRING_CONTENT, "$b c ", # HUH?
|
596
|
+
:tSTRING_DBEG, "\#{",
|
597
|
+
:tSTRING_CONTENT, "3} \n", # HUH?
|
598
|
+
:tSTRING_END, "EOF",
|
599
|
+
:tNL, nil)
|
600
|
+
end
|
601
|
+
|
602
|
+
def test_yylex_heredoc_none
|
603
|
+
util_lex_token("a = <<EOF\nblah\nblah\nEOF",
|
604
|
+
:tIDENTIFIER, "a",
|
605
|
+
:tEQL, "=",
|
606
|
+
:tSTRING_BEG, "\"",
|
607
|
+
:tSTRING_CONTENT, "blah\nblah\n",
|
608
|
+
:tSTRING_CONTENT, "",
|
609
|
+
:tSTRING_END, "EOF",
|
610
|
+
:tNL, nil)
|
611
|
+
end
|
612
|
+
|
613
|
+
def test_yylex_heredoc_none_bad_eos
|
614
|
+
util_bad_token("a = <<EOF",
|
615
|
+
:tIDENTIFIER, "a",
|
616
|
+
:tEQL, "=",
|
617
|
+
:tSTRING_BEG, "\"")
|
618
|
+
end
|
619
|
+
|
620
|
+
def test_yylex_heredoc_none_dash
|
621
|
+
util_lex_token("a = <<-EOF\nblah\nblah\n EOF",
|
622
|
+
:tIDENTIFIER, "a",
|
623
|
+
:tEQL, "=",
|
624
|
+
:tSTRING_BEG, "\"",
|
625
|
+
:tSTRING_CONTENT, "blah\nblah\n",
|
626
|
+
:tSTRING_CONTENT, "",
|
627
|
+
:tSTRING_END, "EOF",
|
628
|
+
:tNL, nil)
|
629
|
+
end
|
630
|
+
|
631
|
+
def test_yylex_heredoc_single
|
632
|
+
util_lex_token("a = <<'EOF'\n blah blah\nEOF\n",
|
633
|
+
:tIDENTIFIER, "a",
|
634
|
+
:tEQL, "=",
|
635
|
+
:tSTRING_BEG, "\"",
|
636
|
+
:tSTRING_CONTENT, " blah blah\n",
|
637
|
+
:tSTRING_END, "EOF",
|
638
|
+
:tNL, nil)
|
639
|
+
end
|
640
|
+
|
641
|
+
def test_yylex_heredoc_single_bad_eos_body
|
642
|
+
util_bad_token("a = <<'EOF'\nblah",
|
643
|
+
:tIDENTIFIER, "a",
|
644
|
+
:tEQL, "=",
|
645
|
+
:tSTRING_BEG, "\"")
|
646
|
+
end
|
647
|
+
|
648
|
+
def test_yylex_heredoc_single_bad_eos_empty
|
649
|
+
util_bad_token("a = <<''\n",
|
650
|
+
:tIDENTIFIER, "a",
|
651
|
+
:tEQL, "=",
|
652
|
+
:tSTRING_BEG, "\"")
|
653
|
+
end
|
654
|
+
|
655
|
+
def test_yylex_heredoc_single_bad_eos_term
|
656
|
+
util_bad_token("a = <<'EOF",
|
657
|
+
:tIDENTIFIER, "a",
|
658
|
+
:tEQL, "=",
|
659
|
+
:tSTRING_BEG, "\"")
|
660
|
+
end
|
661
|
+
|
662
|
+
def test_yylex_heredoc_single_bad_eos_term_nl
|
663
|
+
util_bad_token("a = <<'EOF\ns = 'blah blah'",
|
664
|
+
:tIDENTIFIER, "a",
|
665
|
+
:tEQL, "=",
|
666
|
+
:tSTRING_BEG, "\"")
|
667
|
+
end
|
668
|
+
|
669
|
+
def test_yylex_heredoc_single_dash
|
670
|
+
util_lex_token("a = <<-'EOF'\n blah blah\n EOF\n",
|
671
|
+
:tIDENTIFIER, "a",
|
672
|
+
:tEQL, "=",
|
673
|
+
:tSTRING_BEG, "\"",
|
674
|
+
:tSTRING_CONTENT, " blah blah\n",
|
675
|
+
:tSTRING_END, "EOF",
|
676
|
+
:tNL, nil)
|
677
|
+
end
|
678
|
+
|
679
|
+
def test_yylex_identifier
|
680
|
+
util_lex_token("identifier", :tIDENTIFIER, "identifier")
|
681
|
+
end
|
682
|
+
|
683
|
+
def test_yylex_identifier_bang
|
684
|
+
util_lex_token("identifier!", :tFID, "identifier!")
|
685
|
+
end
|
686
|
+
|
687
|
+
def test_yylex_identifier_cmp
|
688
|
+
util_lex_fname "<=>", :tCMP
|
689
|
+
end
|
690
|
+
|
691
|
+
def test_yylex_identifier_def
|
692
|
+
util_lex_fname "identifier", :tIDENTIFIER, :expr_end
|
693
|
+
end
|
694
|
+
|
695
|
+
def test_yylex_identifier_eh
|
696
|
+
util_lex_token("identifier?", :tFID, "identifier?")
|
697
|
+
end
|
698
|
+
|
699
|
+
def test_yylex_identifier_equals_arrow
|
700
|
+
@lex.lex_state = :expr_fname
|
701
|
+
util_lex_token(":blah==>",
|
702
|
+
:tSYMBOL, "blah=",
|
703
|
+
:tASSOC, "=>")
|
704
|
+
end
|
705
|
+
|
706
|
+
def test_yylex_identifier_equals_caret
|
707
|
+
util_lex_fname "^", :tCARET
|
708
|
+
end
|
709
|
+
|
710
|
+
def test_yylex_identifier_equals_def
|
711
|
+
util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
|
712
|
+
end
|
713
|
+
|
714
|
+
def test_yylex_identifier_equals_def2
|
715
|
+
util_lex_fname "==", :tEQ
|
716
|
+
end
|
717
|
+
|
718
|
+
def test_yylex_identifier_equals_expr
|
719
|
+
@lex.lex_state = :expr_dot
|
720
|
+
util_lex_token("y = arg",
|
721
|
+
:tIDENTIFIER, "y",
|
722
|
+
:tEQL, "=",
|
723
|
+
:tIDENTIFIER, "arg")
|
724
|
+
|
725
|
+
assert_equal :expr_arg, @lex.lex_state
|
726
|
+
end
|
727
|
+
|
728
|
+
def test_yylex_identifier_equals_or
|
729
|
+
util_lex_fname "|", :tPIPE
|
730
|
+
end
|
731
|
+
|
732
|
+
def test_yylex_identifier_equals_slash
|
733
|
+
util_lex_fname "/", :tDIVIDE
|
734
|
+
end
|
735
|
+
|
736
|
+
def test_yylex_identifier_equals_tilde
|
737
|
+
@lex.lex_state = :expr_fname # can only set via parser's defs
|
738
|
+
util_lex_token("identifier=~",
|
739
|
+
:tIDENTIFIER, "identifier",
|
740
|
+
:tMATCH, "=~")
|
741
|
+
end
|
742
|
+
|
743
|
+
def test_yylex_identifier_gt
|
744
|
+
util_lex_fname ">", :tGT
|
745
|
+
end
|
746
|
+
|
747
|
+
def test_yylex_identifier_le
|
748
|
+
util_lex_fname "<=", :tLEQ
|
749
|
+
end
|
750
|
+
|
751
|
+
def test_yylex_identifier_lt
|
752
|
+
util_lex_fname "<", :tLT
|
753
|
+
end
|
754
|
+
|
755
|
+
def test_yylex_identifier_tilde
|
756
|
+
util_lex_fname "~", :tTILDE
|
757
|
+
end
|
758
|
+
|
759
|
+
def test_yylex_index
|
760
|
+
util_lex_fname "[]", :tAREF
|
761
|
+
end
|
762
|
+
|
763
|
+
def test_yylex_index_equals
|
764
|
+
util_lex_fname "[]=", :tASET
|
765
|
+
end
|
766
|
+
|
767
|
+
def test_yylex_integer
|
768
|
+
util_lex_token "42", :tINTEGER, 42
|
769
|
+
end
|
770
|
+
|
771
|
+
def test_yylex_integer_bin
|
772
|
+
util_lex_token "0b101010", :tINTEGER, 42
|
773
|
+
end
|
774
|
+
|
775
|
+
def test_yylex_integer_bin_bad_none
|
776
|
+
util_bad_token "0b "
|
777
|
+
end
|
778
|
+
|
779
|
+
def test_yylex_integer_bin_bad_underscores
|
780
|
+
util_bad_token "0b10__01"
|
781
|
+
end
|
782
|
+
|
783
|
+
def test_yylex_integer_dec
|
784
|
+
util_lex_token "42", :tINTEGER, 42
|
785
|
+
end
|
786
|
+
|
787
|
+
def test_yylex_integer_dec_bad_underscores
|
788
|
+
util_bad_token "42__24"
|
789
|
+
end
|
790
|
+
|
791
|
+
def test_yylex_integer_dec_d
|
792
|
+
util_lex_token "0d42", :tINTEGER, 42
|
793
|
+
end
|
794
|
+
|
795
|
+
def test_yylex_integer_dec_d_bad_none
|
796
|
+
util_bad_token "0d"
|
797
|
+
end
|
798
|
+
|
799
|
+
def test_yylex_integer_dec_d_bad_underscores
|
800
|
+
util_bad_token "0d42__24"
|
801
|
+
end
|
802
|
+
|
803
|
+
def test_yylex_integer_eh_a
|
804
|
+
util_lex_token '?a', :tINTEGER, 97
|
805
|
+
end
|
806
|
+
|
807
|
+
def test_yylex_integer_eh_escape_M_escape_C
|
808
|
+
util_lex_token '?\M-\C-a', :tINTEGER, 129
|
809
|
+
end
|
810
|
+
|
811
|
+
def test_yylex_integer_hex
|
812
|
+
util_lex_token "0x2a", :tINTEGER, 42
|
813
|
+
end
|
814
|
+
|
815
|
+
def test_yylex_integer_hex_bad_none
|
816
|
+
util_bad_token "0x "
|
817
|
+
end
|
818
|
+
|
819
|
+
def test_yylex_integer_hex_bad_underscores
|
820
|
+
util_bad_token "0xab__cd"
|
821
|
+
end
|
822
|
+
|
823
|
+
def test_yylex_integer_oct
|
824
|
+
util_lex_token "052", :tINTEGER, 42
|
825
|
+
end
|
826
|
+
|
827
|
+
def test_yylex_integer_oct_bad_range
|
828
|
+
util_bad_token "08"
|
829
|
+
end
|
830
|
+
|
831
|
+
def test_yylex_integer_oct_bad_underscores
|
832
|
+
util_bad_token "01__23"
|
833
|
+
end
|
834
|
+
|
835
|
+
def test_yylex_integer_oct_o
|
836
|
+
util_lex_token "0o52", :tINTEGER, 42
|
837
|
+
end
|
838
|
+
|
839
|
+
def test_yylex_integer_oct_o_bad_range
|
840
|
+
util_bad_token "0o8"
|
841
|
+
end
|
842
|
+
|
843
|
+
def test_yylex_integer_oct_o_bad_underscores
|
844
|
+
util_bad_token "0o1__23"
|
845
|
+
end
|
846
|
+
|
847
|
+
def test_yylex_integer_oct_o_not_bad_none
|
848
|
+
util_lex_token "0o ", :tINTEGER, 0
|
849
|
+
end
|
850
|
+
|
851
|
+
def test_yylex_integer_trailing
|
852
|
+
util_lex_token("1.to_s",
|
853
|
+
:tINTEGER, 1,
|
854
|
+
:tDOT, '.',
|
855
|
+
:tIDENTIFIER, 'to_s')
|
856
|
+
end
|
857
|
+
|
858
|
+
def test_yylex_integer_underscore
|
859
|
+
util_lex_token "4_2", :tINTEGER, 42
|
860
|
+
end
|
861
|
+
|
862
|
+
def test_yylex_integer_underscore_bad
|
863
|
+
util_bad_token "4__2"
|
864
|
+
end
|
865
|
+
|
866
|
+
def test_yylex_integer_zero
|
867
|
+
util_lex_token "0", :tINTEGER, 0
|
868
|
+
end
|
869
|
+
|
870
|
+
def test_yylex_ivar
|
871
|
+
util_lex_token "@blah", :tIVAR, "@blah"
|
872
|
+
end
|
873
|
+
|
874
|
+
def test_yylex_ivar_bad
|
875
|
+
util_bad_token "@1"
|
37
876
|
end
|
38
877
|
|
39
|
-
def
|
40
|
-
|
41
|
-
|
42
|
-
|
878
|
+
def test_yylex_keyword_expr
|
879
|
+
@lex.lex_state = :expr_endarg
|
880
|
+
|
881
|
+
util_lex_token("if", :kIF_MOD, "if")
|
882
|
+
|
883
|
+
assert_equal :expr_beg, @lex.lex_state
|
43
884
|
end
|
44
885
|
|
45
|
-
def
|
46
|
-
|
47
|
-
node = @lex.parse_number('1')
|
48
|
-
assert_equal :tINTEGER, node
|
49
|
-
assert_equal 142, @lex.yacc_value
|
886
|
+
def test_yylex_lt
|
887
|
+
util_lex_token "<", :tLT, "<"
|
50
888
|
end
|
51
889
|
|
52
|
-
def
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
890
|
+
def test_yylex_lt2
|
891
|
+
util_lex_token("a <\< b",
|
892
|
+
:tIDENTIFIER, "a",
|
893
|
+
:tLSHFT, "<\<",
|
894
|
+
:tIDENTIFIER, "b")
|
895
|
+
|
58
896
|
end
|
59
897
|
|
60
|
-
def
|
61
|
-
util_lex_token "
|
898
|
+
def test_yylex_lt2_equals
|
899
|
+
util_lex_token("a <\<= b",
|
900
|
+
:tIDENTIFIER, "a",
|
901
|
+
:tOP_ASGN, "<\<",
|
902
|
+
:tIDENTIFIER, "b")
|
62
903
|
end
|
63
904
|
|
64
|
-
def
|
65
|
-
util_lex_token
|
66
|
-
:tINTEGER, 97)
|
905
|
+
def test_yylex_lt_equals
|
906
|
+
util_lex_token "<=", :tLEQ, "<="
|
67
907
|
end
|
68
908
|
|
69
|
-
def
|
70
|
-
util_lex_token(
|
71
|
-
:tINTEGER,
|
909
|
+
def test_yylex_minus
|
910
|
+
util_lex_token("1 - 2",
|
911
|
+
:tINTEGER, 1,
|
912
|
+
:tMINUS, "-",
|
913
|
+
:tINTEGER, 2)
|
72
914
|
end
|
73
915
|
|
74
|
-
def
|
75
|
-
util_lex_token "
|
916
|
+
def test_yylex_minus_equals
|
917
|
+
util_lex_token "-=", :tOP_ASGN, "-"
|
76
918
|
end
|
77
919
|
|
78
|
-
def
|
79
|
-
|
80
|
-
|
920
|
+
def test_yylex_minus_method
|
921
|
+
@lex.lex_state = :expr_fname
|
922
|
+
util_lex_token "-", :tMINUS, "-"
|
81
923
|
end
|
82
924
|
|
83
|
-
def
|
84
|
-
|
85
|
-
|
86
|
-
";", t(";"))
|
925
|
+
def test_yylex_minus_unary_method
|
926
|
+
@lex.lex_state = :expr_fname
|
927
|
+
util_lex_token "-@", :tUMINUS, "-@"
|
87
928
|
end
|
88
929
|
|
89
|
-
def
|
90
|
-
util_lex_token("
|
91
|
-
:
|
930
|
+
def test_yylex_minus_unary_number
|
931
|
+
util_lex_token("-42",
|
932
|
+
:tUMINUS_NUM, "-",
|
933
|
+
:tINTEGER, 42)
|
934
|
+
end
|
935
|
+
|
936
|
+
def test_yylex_nth_ref
|
937
|
+
util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
|
938
|
+
:tLBRACK, "[",
|
939
|
+
:tNTH_REF, 1, :tCOMMA, ",",
|
940
|
+
:tNTH_REF, 2, :tCOMMA, ",",
|
941
|
+
:tNTH_REF, 3, :tCOMMA, ",",
|
942
|
+
:tNTH_REF, 4, :tCOMMA, ",",
|
943
|
+
:tNTH_REF, 5, :tCOMMA, ",",
|
944
|
+
:tNTH_REF, 6, :tCOMMA, ",",
|
945
|
+
:tNTH_REF, 7, :tCOMMA, ",",
|
946
|
+
:tNTH_REF, 8, :tCOMMA, ",",
|
947
|
+
:tNTH_REF, 9,
|
948
|
+
:tRBRACK, "]")
|
949
|
+
end
|
950
|
+
|
951
|
+
def test_yylex_open_bracket
|
952
|
+
util_lex_token("(", :tLPAREN, "(")
|
953
|
+
end
|
954
|
+
|
955
|
+
def test_yylex_open_bracket_cmdarg
|
956
|
+
@lex.lex_state = :expr_cmdarg
|
957
|
+
util_lex_token(" (", :tLPAREN_ARG, "(")
|
958
|
+
end
|
959
|
+
|
960
|
+
def test_yylex_open_bracket_exprarg
|
961
|
+
@lex.lex_state = :expr_arg
|
962
|
+
util_lex_token(" (", :tLPAREN2, "(")
|
963
|
+
end
|
964
|
+
|
965
|
+
def test_yylex_open_curly_bracket
|
966
|
+
util_lex_token("{",
|
967
|
+
:tLBRACE, "{")
|
968
|
+
end
|
969
|
+
|
970
|
+
def test_yylex_open_curly_bracket_arg
|
971
|
+
@lex.lex_state = :expr_arg
|
972
|
+
util_lex_token("m { 3 }",
|
973
|
+
:tIDENTIFIER, "m",
|
974
|
+
:tLCURLY, "{",
|
975
|
+
:tINTEGER, 3,
|
976
|
+
:tRCURLY, "}")
|
977
|
+
end
|
978
|
+
|
979
|
+
def test_yylex_open_curly_bracket_block
|
980
|
+
@lex.lex_state = :expr_endarg # seen m(3)
|
981
|
+
util_lex_token("{ 4 }",
|
982
|
+
:tLBRACE_ARG, "{",
|
983
|
+
:tINTEGER, 4,
|
984
|
+
:tRCURLY, "}")
|
985
|
+
end
|
986
|
+
|
987
|
+
def test_yylex_open_square_bracket_arg
|
988
|
+
@lex.lex_state = :expr_arg
|
989
|
+
util_lex_token("m [ 3 ]",
|
990
|
+
:tIDENTIFIER, "m",
|
991
|
+
:tLBRACK, "[",
|
992
|
+
:tINTEGER, 3,
|
993
|
+
:tRBRACK, "]")
|
994
|
+
end
|
995
|
+
|
996
|
+
def test_yylex_open_square_bracket_ary
|
997
|
+
util_lex_token("[1, 2, 3]",
|
998
|
+
:tLBRACK, "[",
|
999
|
+
:tINTEGER, 1,
|
1000
|
+
:tCOMMA, ",",
|
1001
|
+
:tINTEGER, 2,
|
1002
|
+
:tCOMMA, ",",
|
1003
|
+
:tINTEGER, 3,
|
1004
|
+
:tRBRACK, "]")
|
1005
|
+
end
|
1006
|
+
|
1007
|
+
def test_yylex_open_square_bracket_meth
|
1008
|
+
util_lex_token("m[3]",
|
1009
|
+
:tIDENTIFIER, "m",
|
1010
|
+
"[", "[",
|
1011
|
+
:tINTEGER, 3,
|
1012
|
+
:tRBRACK, "]")
|
1013
|
+
end
|
1014
|
+
|
1015
|
+
def test_yylex_or
|
1016
|
+
util_lex_token "|", :tPIPE, "|"
|
1017
|
+
end
|
1018
|
+
|
1019
|
+
def test_yylex_or2
|
1020
|
+
util_lex_token "||", :tOROP, "||"
|
1021
|
+
end
|
1022
|
+
|
1023
|
+
def test_yylex_or2_equals
|
1024
|
+
util_lex_token "||=", :tOP_ASGN, "||"
|
1025
|
+
end
|
1026
|
+
|
1027
|
+
def test_yylex_or_equals
|
1028
|
+
util_lex_token "|=", :tOP_ASGN, "|"
|
1029
|
+
end
|
1030
|
+
|
1031
|
+
def test_yylex_percent
|
1032
|
+
util_lex_token("a % 2",
|
1033
|
+
:tIDENTIFIER, "a",
|
1034
|
+
:tPERCENT, "%",
|
1035
|
+
:tINTEGER, 2)
|
1036
|
+
end
|
1037
|
+
|
1038
|
+
def test_yylex_percent_equals
|
1039
|
+
util_lex_token("a %= 2",
|
1040
|
+
:tIDENTIFIER, "a",
|
1041
|
+
:tOP_ASGN, "%",
|
1042
|
+
:tINTEGER, 2)
|
1043
|
+
end
|
1044
|
+
|
1045
|
+
def test_yylex_plus
|
1046
|
+
util_lex_token("1 + 1", # TODO lex_state?
|
1047
|
+
:tINTEGER, 1,
|
1048
|
+
:tPLUS, "+",
|
1049
|
+
:tINTEGER, 1)
|
1050
|
+
end
|
1051
|
+
|
1052
|
+
def test_yylex_plus_equals
|
1053
|
+
util_lex_token "+=", :tOP_ASGN, "+"
|
1054
|
+
end
|
1055
|
+
|
1056
|
+
def test_yylex_plus_method
|
1057
|
+
@lex.lex_state = :expr_fname
|
1058
|
+
util_lex_token "+", :tPLUS, "+"
|
1059
|
+
end
|
1060
|
+
|
1061
|
+
def test_yylex_plus_unary_method
|
1062
|
+
@lex.lex_state = :expr_fname
|
1063
|
+
util_lex_token "+@", :tUPLUS, "+@"
|
1064
|
+
end
|
1065
|
+
|
1066
|
+
def test_yylex_plus_unary_number
|
1067
|
+
util_lex_token("+42",
|
1068
|
+
:tINTEGER, 42)
|
1069
|
+
end
|
1070
|
+
|
1071
|
+
def test_yylex_question
|
1072
|
+
util_lex_token "?*", :tINTEGER, 42
|
1073
|
+
end
|
1074
|
+
|
1075
|
+
def test_yylex_question_bad_eos
|
1076
|
+
util_bad_token "?"
|
1077
|
+
end
|
1078
|
+
|
1079
|
+
def test_yylex_question_ws
|
1080
|
+
util_lex_token "? ", :tEH, "?"
|
1081
|
+
util_lex_token "?\n", :tEH, "?"
|
1082
|
+
util_lex_token "?\t", :tEH, "?"
|
1083
|
+
util_lex_token "?\v", :tEH, "?"
|
1084
|
+
util_lex_token "?\r", :tEH, "?"
|
1085
|
+
util_lex_token "?\f", :tEH, "?"
|
1086
|
+
end
|
1087
|
+
|
1088
|
+
def test_yylex_question_ws_backslashed
|
1089
|
+
@lex.lex_state = :expr_beg
|
1090
|
+
util_lex_token "?\\ ", :tINTEGER, 32
|
1091
|
+
@lex.lex_state = :expr_beg
|
1092
|
+
util_lex_token "?\\n", :tINTEGER, 10
|
1093
|
+
@lex.lex_state = :expr_beg
|
1094
|
+
util_lex_token "?\\t", :tINTEGER, 9
|
1095
|
+
@lex.lex_state = :expr_beg
|
1096
|
+
util_lex_token "?\\v", :tINTEGER, 11
|
1097
|
+
@lex.lex_state = :expr_beg
|
1098
|
+
util_lex_token "?\\r", :tINTEGER, 13
|
1099
|
+
@lex.lex_state = :expr_beg
|
1100
|
+
util_lex_token "?\\f", :tINTEGER, 12
|
1101
|
+
end
|
1102
|
+
|
1103
|
+
def test_yylex_rbracket
|
1104
|
+
util_lex_token "]", :tRBRACK, "]"
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
def test_yylex_rcurly
|
1108
|
+
util_lex_token "}", :tRCURLY, "}"
|
92
1109
|
end
|
93
1110
|
|
94
1111
|
def test_yylex_regexp
|
95
1112
|
util_lex_token("/regexp/",
|
96
|
-
:tREGEXP_BEG,
|
97
|
-
:tSTRING_CONTENT,
|
1113
|
+
:tREGEXP_BEG, "/",
|
1114
|
+
:tSTRING_CONTENT, "regexp",
|
98
1115
|
:tREGEXP_END, "")
|
99
1116
|
end
|
100
1117
|
|
101
|
-
def
|
102
|
-
util_lex_token("
|
103
|
-
:
|
104
|
-
:
|
105
|
-
:
|
1118
|
+
def test_yylex_regexp_ambiguous
|
1119
|
+
util_lex_token("method /regexp/",
|
1120
|
+
:tIDENTIFIER, "method",
|
1121
|
+
:tREGEXP_BEG, "/",
|
1122
|
+
:tSTRING_CONTENT, "regexp",
|
1123
|
+
:tREGEXP_END, "")
|
1124
|
+
end
|
1125
|
+
|
1126
|
+
def test_yylex_regexp_bad
|
1127
|
+
util_bad_token("/.*/xyz",
|
1128
|
+
:tREGEXP_BEG, "/",
|
1129
|
+
:tSTRING_CONTENT, ".*")
|
1130
|
+
end
|
1131
|
+
|
1132
|
+
def test_yylex_regexp_escape_C
|
1133
|
+
util_lex_token('/regex\\C-x/',
|
1134
|
+
:tREGEXP_BEG, "/",
|
1135
|
+
:tSTRING_CONTENT, "regex\\C-x",
|
1136
|
+
:tREGEXP_END, "")
|
1137
|
+
end
|
1138
|
+
|
1139
|
+
def test_yylex_regexp_escape_C_M
|
1140
|
+
util_lex_token('/regex\\C-\\M-x/',
|
1141
|
+
:tREGEXP_BEG, "/",
|
1142
|
+
:tSTRING_CONTENT, "regex\\C-\\M-x",
|
1143
|
+
:tREGEXP_END, "")
|
1144
|
+
end
|
1145
|
+
|
1146
|
+
def test_yylex_regexp_escape_C_M_craaaazy
|
1147
|
+
util_lex_token("/regex\\C-\\\n\\M-x/",
|
1148
|
+
:tREGEXP_BEG, "/",
|
1149
|
+
:tSTRING_CONTENT, "regex\\C-\\M-x",
|
1150
|
+
:tREGEXP_END, "")
|
1151
|
+
end
|
1152
|
+
|
1153
|
+
def test_yylex_regexp_escape_C_bad_dash
|
1154
|
+
util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
|
1155
|
+
end
|
1156
|
+
|
1157
|
+
def test_yylex_regexp_escape_C_bad_dash_eos
|
1158
|
+
util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
|
1159
|
+
end
|
1160
|
+
|
1161
|
+
def test_yylex_regexp_escape_C_bad_dash_eos2
|
1162
|
+
util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
|
1163
|
+
end
|
1164
|
+
|
1165
|
+
def test_yylex_regexp_escape_C_bad_eos
|
1166
|
+
util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
|
1167
|
+
end
|
1168
|
+
|
1169
|
+
def test_yylex_regexp_escape_C_bad_eos2
|
1170
|
+
util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
|
1171
|
+
end
|
1172
|
+
|
1173
|
+
def test_yylex_regexp_escape_M
|
1174
|
+
util_lex_token('/regex\\M-x/',
|
1175
|
+
:tREGEXP_BEG, "/",
|
1176
|
+
:tSTRING_CONTENT, "regex\\M-x",
|
1177
|
+
:tREGEXP_END, "")
|
1178
|
+
end
|
1179
|
+
|
1180
|
+
def test_yylex_regexp_escape_M_C
|
1181
|
+
util_lex_token('/regex\\M-\\C-x/',
|
1182
|
+
:tREGEXP_BEG, "/",
|
1183
|
+
:tSTRING_CONTENT, "regex\\M-\\C-x",
|
1184
|
+
:tREGEXP_END, "")
|
1185
|
+
end
|
1186
|
+
|
1187
|
+
def test_yylex_regexp_escape_M_bad_dash
|
1188
|
+
util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
|
1189
|
+
end
|
1190
|
+
|
1191
|
+
def test_yylex_regexp_escape_M_bad_dash_eos
|
1192
|
+
util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
|
1193
|
+
end
|
1194
|
+
|
1195
|
+
def test_yylex_regexp_escape_M_bad_dash_eos2
|
1196
|
+
util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
|
1197
|
+
end
|
1198
|
+
|
1199
|
+
def test_yylex_regexp_escape_M_bad_eos
|
1200
|
+
util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
|
1201
|
+
end
|
1202
|
+
|
1203
|
+
def test_yylex_regexp_escape_backslash_slash
|
1204
|
+
util_lex_token('/\\//',
|
1205
|
+
:tREGEXP_BEG, "/",
|
1206
|
+
:tSTRING_CONTENT, '\\/',
|
1207
|
+
:tREGEXP_END, "")
|
1208
|
+
end
|
1209
|
+
|
1210
|
+
def test_yylex_regexp_escape_backslash_terminator
|
1211
|
+
util_lex_token('%r%blah\\%blah%',
|
1212
|
+
:tREGEXP_BEG, "%r\000", # FIX ?!?
|
1213
|
+
:tSTRING_CONTENT, "blah\\%blah",
|
1214
|
+
:tREGEXP_END, "")
|
1215
|
+
end
|
1216
|
+
|
1217
|
+
def test_yylex_regexp_escape_backslash_terminator_meta1
|
1218
|
+
util_lex_token('%r{blah\\}blah}',
|
1219
|
+
:tREGEXP_BEG, "%r{", # FIX ?!?
|
1220
|
+
:tSTRING_CONTENT, "blah\\}blah",
|
1221
|
+
:tREGEXP_END, "")
|
1222
|
+
end
|
1223
|
+
|
1224
|
+
def test_yylex_regexp_escape_backslash_terminator_meta2
|
1225
|
+
util_lex_token('%r/blah\\/blah/',
|
1226
|
+
:tREGEXP_BEG, "%r\000", # FIX ?!?
|
1227
|
+
:tSTRING_CONTENT, "blah\\/blah",
|
1228
|
+
:tREGEXP_END, "")
|
1229
|
+
end
|
1230
|
+
|
1231
|
+
def test_yylex_regexp_escape_backslash_terminator_meta3
|
1232
|
+
util_lex_token('%r/blah\\%blah/',
|
1233
|
+
:tREGEXP_BEG, "%r\000", # FIX ?!?
|
1234
|
+
:tSTRING_CONTENT, "blah\\%blah",
|
1235
|
+
:tREGEXP_END, "")
|
1236
|
+
end
|
1237
|
+
|
1238
|
+
def test_yylex_regexp_escape_bad_eos
|
1239
|
+
util_bad_token '/regex\\', :tREGEXP_BEG, "/"
|
1240
|
+
end
|
1241
|
+
|
1242
|
+
def test_yylex_regexp_escape_bs
|
1243
|
+
util_lex_token('/regex\\\\regex/',
|
1244
|
+
:tREGEXP_BEG, "/",
|
1245
|
+
:tSTRING_CONTENT, "regex\\\\regex",
|
1246
|
+
:tREGEXP_END, "")
|
1247
|
+
end
|
1248
|
+
|
1249
|
+
def test_yylex_regexp_escape_c
|
1250
|
+
util_lex_token('/regex\\cxxx/',
|
1251
|
+
:tREGEXP_BEG, "/",
|
1252
|
+
:tSTRING_CONTENT, "regex\\cxxx",
|
1253
|
+
:tREGEXP_END, "")
|
106
1254
|
end
|
107
1255
|
|
108
|
-
def
|
109
|
-
util_lex_token('/
|
110
|
-
:tREGEXP_BEG,
|
111
|
-
:tSTRING_CONTENT,
|
1256
|
+
def test_yylex_regexp_escape_c_backslash
|
1257
|
+
util_lex_token('/regex\\c\\n/',
|
1258
|
+
:tREGEXP_BEG, "/",
|
1259
|
+
:tSTRING_CONTENT, "regex\\c\\n",
|
112
1260
|
:tREGEXP_END, "")
|
113
1261
|
end
|
114
1262
|
|
115
|
-
def
|
116
|
-
util_lex_token('/re
|
117
|
-
:tREGEXP_BEG,
|
118
|
-
:tSTRING_CONTENT,
|
1263
|
+
def test_yylex_regexp_escape_chars
|
1264
|
+
util_lex_token('/re\\tge\\nxp/',
|
1265
|
+
:tREGEXP_BEG, "/",
|
1266
|
+
:tSTRING_CONTENT, "re\\tge\\nxp",
|
1267
|
+
:tREGEXP_END, "")
|
1268
|
+
end
|
1269
|
+
|
1270
|
+
def test_yylex_regexp_escape_double_backslash
|
1271
|
+
regexp = '/[\\/\\\\]$/'
|
1272
|
+
util_lex_token(regexp,
|
1273
|
+
:tREGEXP_BEG, "/",
|
1274
|
+
:tSTRING_CONTENT, regexp[1..-2],
|
119
1275
|
:tREGEXP_END, "")
|
120
1276
|
end
|
121
1277
|
|
122
1278
|
def test_yylex_regexp_escape_hex
|
123
|
-
util_lex_token('/
|
124
|
-
:tREGEXP_BEG,
|
125
|
-
:tSTRING_CONTENT,
|
1279
|
+
util_lex_token('/regex\\x61xp/',
|
1280
|
+
:tREGEXP_BEG, "/",
|
1281
|
+
:tSTRING_CONTENT, "regex\\x61xp",
|
1282
|
+
:tREGEXP_END, "")
|
1283
|
+
end
|
1284
|
+
|
1285
|
+
def test_yylex_regexp_escape_hex_one
|
1286
|
+
util_lex_token('/^[\\xd\\xa]{2}/on',
|
1287
|
+
:tREGEXP_BEG, '/',
|
1288
|
+
:tSTRING_CONTENT, '^[\\xd\\xa]{2}',
|
1289
|
+
:tREGEXP_END, 'on')
|
1290
|
+
end
|
1291
|
+
|
1292
|
+
def test_yylex_regexp_escape_hex_bad
|
1293
|
+
util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
|
1294
|
+
end
|
1295
|
+
|
1296
|
+
def test_yylex_regexp_escape_oct1
|
1297
|
+
util_lex_token('/regex\\0xp/',
|
1298
|
+
:tREGEXP_BEG, "/",
|
1299
|
+
:tSTRING_CONTENT, "regex\\0xp",
|
1300
|
+
:tREGEXP_END, "")
|
1301
|
+
end
|
1302
|
+
|
1303
|
+
def test_yylex_regexp_escape_oct2
|
1304
|
+
util_lex_token('/regex\\07xp/',
|
1305
|
+
:tREGEXP_BEG, "/",
|
1306
|
+
:tSTRING_CONTENT, "regex\\07xp",
|
1307
|
+
:tREGEXP_END, "")
|
1308
|
+
end
|
1309
|
+
|
1310
|
+
def test_yylex_regexp_escape_oct3
|
1311
|
+
util_lex_token('/regex\\10142/',
|
1312
|
+
:tREGEXP_BEG, "/",
|
1313
|
+
:tSTRING_CONTENT, "regex\\10142",
|
126
1314
|
:tREGEXP_END, "")
|
127
1315
|
end
|
128
1316
|
|
1317
|
+
def test_yylex_regexp_escape_return
|
1318
|
+
util_lex_token("/regex\\\nregex/",
|
1319
|
+
:tREGEXP_BEG, "/",
|
1320
|
+
:tSTRING_CONTENT, "regexregex",
|
1321
|
+
:tREGEXP_END, "")
|
1322
|
+
end
|
1323
|
+
|
1324
|
+
def test_yylex_regexp_nm
|
1325
|
+
util_lex_token("/.*/nm",
|
1326
|
+
:tREGEXP_BEG, "/",
|
1327
|
+
:tSTRING_CONTENT, ".*",
|
1328
|
+
:tREGEXP_END, "nm")
|
1329
|
+
end
|
1330
|
+
|
1331
|
+
def test_yylex_rparen
|
1332
|
+
util_lex_token ")", :tRPAREN, ")"
|
1333
|
+
end
|
1334
|
+
|
1335
|
+
def test_yylex_rshft
|
1336
|
+
util_lex_token("a >> 2",
|
1337
|
+
:tIDENTIFIER, "a",
|
1338
|
+
:tRSHFT, ">>",
|
1339
|
+
:tINTEGER, 2)
|
1340
|
+
end
|
1341
|
+
|
1342
|
+
def test_yylex_rshft_equals
|
1343
|
+
util_lex_token("a >>= 2",
|
1344
|
+
:tIDENTIFIER, "a",
|
1345
|
+
:tOP_ASGN, ">>",
|
1346
|
+
:tINTEGER, 2)
|
1347
|
+
end
|
1348
|
+
|
1349
|
+
def test_yylex_star
|
1350
|
+
util_lex_token("a * ",
|
1351
|
+
:tIDENTIFIER, "a",
|
1352
|
+
:tSTAR2, "*")
|
1353
|
+
|
1354
|
+
assert_equal :expr_beg, @lex.lex_state
|
1355
|
+
end
|
1356
|
+
|
1357
|
+
def test_yylex_star2
|
1358
|
+
util_lex_token("a ** ",
|
1359
|
+
:tIDENTIFIER, "a",
|
1360
|
+
:tPOW, "**")
|
1361
|
+
|
1362
|
+
assert_equal :expr_beg, @lex.lex_state
|
1363
|
+
end
|
1364
|
+
|
1365
|
+
def test_yylex_star2_equals
|
1366
|
+
util_lex_token("a **= ",
|
1367
|
+
:tIDENTIFIER, "a",
|
1368
|
+
:tOP_ASGN, "**")
|
1369
|
+
|
1370
|
+
assert_equal :expr_beg, @lex.lex_state
|
1371
|
+
end
|
1372
|
+
|
1373
|
+
def test_yylex_star_arg
|
1374
|
+
@lex.lex_state = :expr_arg
|
1375
|
+
|
1376
|
+
util_lex_token(" *a",
|
1377
|
+
:tSTAR, "*",
|
1378
|
+
:tIDENTIFIER, "a")
|
1379
|
+
|
1380
|
+
assert_equal :expr_arg, @lex.lex_state
|
1381
|
+
end
|
1382
|
+
|
1383
|
+
def test_yylex_star_arg_beg
|
1384
|
+
@lex.lex_state = :expr_beg
|
1385
|
+
|
1386
|
+
util_lex_token("*a",
|
1387
|
+
:tSTAR, "*",
|
1388
|
+
:tIDENTIFIER, "a")
|
1389
|
+
|
1390
|
+
assert_equal :expr_arg, @lex.lex_state
|
1391
|
+
end
|
1392
|
+
|
1393
|
+
def test_yylex_star_arg_beg_fname
|
1394
|
+
@lex.lex_state = :expr_fname
|
1395
|
+
|
1396
|
+
util_lex_token("*a",
|
1397
|
+
:tSTAR2, "*",
|
1398
|
+
:tIDENTIFIER, "a")
|
1399
|
+
|
1400
|
+
assert_equal :expr_arg, @lex.lex_state
|
1401
|
+
end
|
1402
|
+
|
1403
|
+
def test_yylex_star_equals
|
1404
|
+
util_lex_token("a *= ",
|
1405
|
+
:tIDENTIFIER, "a",
|
1406
|
+
:tOP_ASGN, "*")
|
1407
|
+
|
1408
|
+
assert_equal :expr_beg, @lex.lex_state
|
1409
|
+
end
|
1410
|
+
|
1411
|
+
def test_yylex_string_bad_eos
|
1412
|
+
util_bad_token('%',
|
1413
|
+
:tSTRING_BEG, '%')
|
1414
|
+
end
|
1415
|
+
|
1416
|
+
def test_yylex_string_bad_eos_quote
|
1417
|
+
util_bad_token('%{nest',
|
1418
|
+
:tSTRING_BEG, '%}')
|
1419
|
+
end
|
1420
|
+
|
129
1421
|
def test_yylex_string_double
|
130
1422
|
util_lex_token('"string"',
|
131
|
-
:
|
132
|
-
:tSTRING_CONTENT, s(:str, "string"),
|
133
|
-
:tSTRING_END, t('"'))
|
1423
|
+
:tSTRING, "string")
|
134
1424
|
end
|
135
1425
|
|
136
|
-
def
|
137
|
-
util_lex_token('"
|
138
|
-
:
|
139
|
-
:tSTRING_CONTENT, s(:str, "s\tri\ng"),
|
140
|
-
:tSTRING_END, t('"'))
|
1426
|
+
def test_yylex_string_double_escape_M
|
1427
|
+
util_lex_token('"\\M-g"',
|
1428
|
+
:tSTRING, "\347")
|
141
1429
|
end
|
142
1430
|
|
143
|
-
def
|
144
|
-
util_lex_token('"
|
145
|
-
:
|
146
|
-
:tSTRING_CONTENT, s(:str, "\347"),
|
147
|
-
:tSTRING_END, t('"'))
|
1431
|
+
def test_yylex_string_escape_x_single
|
1432
|
+
util_lex_token('"\\x0"',
|
1433
|
+
:tSTRING, "\000")
|
148
1434
|
end
|
149
1435
|
|
150
|
-
def
|
151
|
-
util_lex_token('"
|
152
|
-
:
|
153
|
-
:tSTRING_CONTENT, s(:str, "n = ABC"),
|
154
|
-
:tSTRING_END, t('"'))
|
1436
|
+
def test_yylex_string_double_escape_chars
|
1437
|
+
util_lex_token('"s\\tri\\ng"',
|
1438
|
+
:tSTRING, "s\tri\ng")
|
155
1439
|
end
|
156
1440
|
|
157
1441
|
def test_yylex_string_double_escape_hex
|
158
|
-
util_lex_token('"n =
|
159
|
-
:
|
160
|
-
:tSTRING_CONTENT, s(:str, "n = abc"),
|
161
|
-
:tSTRING_END, t('"'))
|
1442
|
+
util_lex_token('"n = \\x61\\x62\\x63"',
|
1443
|
+
:tSTRING, "n = abc")
|
162
1444
|
end
|
163
1445
|
|
164
|
-
def
|
165
|
-
util_lex_token("'
|
166
|
-
:
|
167
|
-
:tSTRING_CONTENT, s(:str, "string"),
|
168
|
-
:tSTRING_END, t("'"))
|
1446
|
+
def test_yylex_string_double_escape_bs1
|
1447
|
+
util_lex_token('"a\\a\\a"',
|
1448
|
+
:tSTRING, "a\a\a")
|
169
1449
|
end
|
170
1450
|
|
171
|
-
def
|
172
|
-
util_lex_token("
|
173
|
-
:
|
174
|
-
:tSTRING_CONTENT, s(:str, "string"),
|
175
|
-
:tSTRING_END, t("]"))
|
1451
|
+
def test_yylex_string_double_escape_bs2
|
1452
|
+
util_lex_token('"a\\\\a"',
|
1453
|
+
:tSTRING, "a\\a")
|
176
1454
|
end
|
177
1455
|
|
178
|
-
def
|
179
|
-
util_lex_token("
|
180
|
-
:
|
181
|
-
:tSTRING_CONTENT, s(:str, "s\\tri\\ng"),
|
182
|
-
:tSTRING_END, t("'"))
|
1456
|
+
def test_yylex_string_double_escape_octal
|
1457
|
+
util_lex_token('"n = \\101\\102\\103"',
|
1458
|
+
:tSTRING, "n = ABC")
|
183
1459
|
end
|
184
1460
|
|
185
|
-
def
|
186
|
-
util_lex_token("
|
187
|
-
:
|
1461
|
+
def test_yylex_string_double_interp
|
1462
|
+
util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
|
1463
|
+
:tSTRING_BEG, "\"",
|
1464
|
+
:tSTRING_CONTENT, "blah #x a ",
|
1465
|
+
:tSTRING_DVAR, nil,
|
1466
|
+
:tSTRING_CONTENT, "@a b ",
|
1467
|
+
:tSTRING_DVAR, nil,
|
1468
|
+
:tSTRING_CONTENT, "$b c ",
|
1469
|
+
:tSTRING_DBEG, nil,
|
1470
|
+
:tSTRING_CONTENT, "3} # ",
|
1471
|
+
:tSTRING_END, "\"")
|
188
1472
|
end
|
189
1473
|
|
190
|
-
def
|
191
|
-
util_lex_token(
|
192
|
-
:
|
1474
|
+
def test_yylex_string_double_nested_curlies
|
1475
|
+
util_lex_token('%{nest{one{two}one}nest}',
|
1476
|
+
:tSTRING_BEG, '%}',
|
1477
|
+
:tSTRING_CONTENT, "nest{one{two}one}nest",
|
1478
|
+
:tSTRING_END, '}')
|
193
1479
|
end
|
194
1480
|
|
195
|
-
def
|
196
|
-
util_lex_token("
|
197
|
-
:
|
1481
|
+
def test_yylex_string_double_no_interp
|
1482
|
+
util_lex_token("\"# blah\"", # pound first
|
1483
|
+
:tSTRING, "# blah")
|
1484
|
+
|
1485
|
+
util_lex_token("\"blah # blah\"", # pound not first
|
1486
|
+
:tSTRING, "blah # blah")
|
198
1487
|
end
|
199
1488
|
|
200
|
-
def
|
201
|
-
util_lex_token("
|
202
|
-
:
|
203
|
-
:
|
1489
|
+
def test_yylex_string_pct_Q
|
1490
|
+
util_lex_token("%Q[s1 s2]",
|
1491
|
+
:tSTRING_BEG, "%Q[",
|
1492
|
+
:tSTRING_CONTENT, "s1 s2",
|
1493
|
+
:tSTRING_END, "]")
|
204
1494
|
end
|
205
1495
|
|
206
|
-
def
|
207
|
-
util_lex_token("
|
208
|
-
:
|
1496
|
+
def test_yylex_string_pct_W
|
1497
|
+
util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
|
1498
|
+
:tWORDS_BEG, "%W[",
|
1499
|
+
:tSTRING_CONTENT, "s1",
|
1500
|
+
:tSPACE, nil,
|
1501
|
+
:tSTRING_CONTENT, "s2",
|
1502
|
+
:tSPACE, nil,
|
1503
|
+
:tSTRING_CONTENT, "s3",
|
1504
|
+
:tSPACE, nil,
|
1505
|
+
:tSTRING_END, nil)
|
209
1506
|
end
|
210
1507
|
|
211
|
-
def
|
212
|
-
|
1508
|
+
def test_yylex_string_pct_W_bs_nl
|
1509
|
+
util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
|
1510
|
+
:tWORDS_BEG, "%W[",
|
1511
|
+
:tSTRING_CONTENT, "s1",
|
1512
|
+
:tSPACE, nil,
|
1513
|
+
:tSTRING_CONTENT, "\ns2",
|
1514
|
+
:tSPACE, nil,
|
1515
|
+
:tSTRING_END, nil)
|
1516
|
+
end
|
213
1517
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
1518
|
+
def test_yylex_string_pct_angle
|
1519
|
+
util_lex_token("%<blah>",
|
1520
|
+
:tSTRING_BEG, "%>",
|
1521
|
+
:tSTRING_CONTENT, "blah",
|
1522
|
+
:tSTRING_END, ">")
|
1523
|
+
end
|
220
1524
|
|
221
|
-
|
1525
|
+
def test_yylex_string_pct_other
|
1526
|
+
util_lex_token("%%blah%",
|
1527
|
+
:tSTRING_BEG, "%%",
|
1528
|
+
:tSTRING_CONTENT, "blah",
|
1529
|
+
:tSTRING_END, "%")
|
222
1530
|
end
|
223
|
-
end
|
224
1531
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
end
|
233
|
-
|
234
|
-
def test_is_in_state
|
235
|
-
s = StackState.new :test
|
236
|
-
assert_equal false, s.is_in_state
|
237
|
-
s.push false
|
238
|
-
assert_equal false, s.is_in_state
|
239
|
-
s.push true
|
240
|
-
assert_equal true, s.is_in_state
|
241
|
-
s.push false
|
242
|
-
assert_equal false, s.is_in_state
|
243
|
-
end
|
244
|
-
|
245
|
-
def test_lexpop
|
246
|
-
s = StackState.new :test
|
247
|
-
assert_equal [false], s.stack
|
248
|
-
s.push true
|
249
|
-
s.push false
|
250
|
-
assert_equal [false, true, false], s.stack
|
251
|
-
s.lexpop
|
252
|
-
assert_equal [false, true], s.stack
|
253
|
-
end
|
254
|
-
|
255
|
-
def test_pop
|
256
|
-
s = StackState.new :test
|
257
|
-
assert_equal [false], s.stack
|
258
|
-
s.push true
|
259
|
-
assert_equal [false, true], s.stack
|
260
|
-
assert_equal true, s.pop
|
261
|
-
assert_equal [false], s.stack
|
262
|
-
end
|
263
|
-
|
264
|
-
def test_push
|
265
|
-
s = StackState.new :test
|
266
|
-
assert_equal [false], s.stack
|
267
|
-
s.push true
|
268
|
-
s.push false
|
269
|
-
assert_equal [false, true, false], s.stack
|
1532
|
+
def test_yylex_string_pct_w
|
1533
|
+
util_bad_token("%w[s1 s2 ",
|
1534
|
+
:tAWORDS_BEG, "%w[",
|
1535
|
+
:tSTRING_CONTENT, "s1",
|
1536
|
+
:tSPACE, nil,
|
1537
|
+
:tSTRING_CONTENT, "s2",
|
1538
|
+
:tSPACE, nil)
|
270
1539
|
end
|
271
|
-
end
|
272
1540
|
|
273
|
-
|
274
|
-
|
275
|
-
|
1541
|
+
def test_yylex_string_pct_w_bs_nl
|
1542
|
+
util_lex_token("%w[s1 \\\ns2]",
|
1543
|
+
:tAWORDS_BEG, "%w[",
|
1544
|
+
:tSTRING_CONTENT, "s1",
|
1545
|
+
:tSPACE, nil,
|
1546
|
+
:tSTRING_CONTENT, "\ns2",
|
1547
|
+
:tSPACE, nil,
|
1548
|
+
:tSTRING_END, nil)
|
276
1549
|
end
|
277
1550
|
|
278
|
-
def
|
279
|
-
|
280
|
-
|
281
|
-
|
1551
|
+
def test_yylex_string_pct_w_bs_sp
|
1552
|
+
util_lex_token("%w[s\\ 1 s\\ 2]",
|
1553
|
+
:tAWORDS_BEG, "%w[",
|
1554
|
+
:tSTRING_CONTENT, "s 1",
|
1555
|
+
:tSPACE, nil,
|
1556
|
+
:tSTRING_CONTENT, "s 2",
|
1557
|
+
:tSPACE, nil,
|
1558
|
+
:tSTRING_END, nil)
|
1559
|
+
end
|
1560
|
+
|
1561
|
+
def test_yylex_string_single
|
1562
|
+
util_lex_token("'string'",
|
1563
|
+
:tSTRING, "string")
|
1564
|
+
end
|
1565
|
+
|
1566
|
+
def test_yylex_string_single_escape_chars
|
1567
|
+
util_lex_token("'s\\tri\\ng'",
|
1568
|
+
:tSTRING, "s\\tri\\ng")
|
1569
|
+
end
|
1570
|
+
|
1571
|
+
def test_yylex_string_single_nl
|
1572
|
+
util_lex_token("'blah\\\nblah'",
|
1573
|
+
:tSTRING, "blah\\\nblah")
|
282
1574
|
end
|
283
1575
|
|
284
|
-
def
|
285
|
-
|
286
|
-
|
287
|
-
assert_equal expected, @env.instance_variable_get(:"@use")
|
1576
|
+
def test_yylex_symbol
|
1577
|
+
util_lex_token(":symbol",
|
1578
|
+
:tSYMBOL, "symbol")
|
288
1579
|
end
|
289
1580
|
|
290
|
-
def
|
291
|
-
|
292
|
-
|
293
|
-
expected = [{}, { :blah => true }]
|
294
|
-
assert_equal expected, @env.instance_variable_get(:"@use")
|
1581
|
+
def test_yylex_symbol_bad_zero
|
1582
|
+
util_bad_token(":\"blah\0\"",
|
1583
|
+
:tSYMBEG, ":")
|
295
1584
|
end
|
296
1585
|
|
297
|
-
def
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
1586
|
+
def test_yylex_symbol_double
|
1587
|
+
util_lex_token(":\"symbol\"",
|
1588
|
+
:tSYMBEG, ":",
|
1589
|
+
:tSTRING_CONTENT, "symbol",
|
1590
|
+
:tSTRING_END, '"')
|
302
1591
|
end
|
303
1592
|
|
304
|
-
def
|
305
|
-
|
1593
|
+
def test_yylex_symbol_single
|
1594
|
+
util_lex_token(":'symbol'",
|
1595
|
+
:tSYMBEG, ":",
|
1596
|
+
:tSTRING_CONTENT, "symbol",
|
1597
|
+
:tSTRING_END, "'")
|
306
1598
|
end
|
307
1599
|
|
308
|
-
def
|
309
|
-
|
310
|
-
|
311
|
-
|
1600
|
+
def test_yylex_ternary
|
1601
|
+
util_lex_token("a ? b : c",
|
1602
|
+
:tIDENTIFIER, "a",
|
1603
|
+
:tEH, "?",
|
1604
|
+
:tIDENTIFIER, "b",
|
1605
|
+
:tCOLON, ":",
|
1606
|
+
:tIDENTIFIER, "c")
|
1607
|
+
|
1608
|
+
util_lex_token("a ?bb : c", # GAH! MATZ!!!
|
1609
|
+
:tIDENTIFIER, "a",
|
1610
|
+
:tEH, "?",
|
1611
|
+
:tIDENTIFIER, "bb",
|
1612
|
+
:tCOLON, ":",
|
1613
|
+
:tIDENTIFIER, "c")
|
1614
|
+
|
1615
|
+
util_lex_token("42 ?", # 42 forces expr_end
|
1616
|
+
:tINTEGER, 42,
|
1617
|
+
:tEH, "?")
|
312
1618
|
end
|
313
1619
|
|
314
|
-
def
|
315
|
-
|
316
|
-
assert_equal 42, @env[:blah]
|
317
|
-
@env.unextend
|
318
|
-
assert_equal 42, @env[:blah]
|
1620
|
+
def test_yylex_tilde
|
1621
|
+
util_lex_token "~", :tTILDE, "~"
|
319
1622
|
end
|
320
1623
|
|
321
|
-
def
|
322
|
-
@
|
323
|
-
|
324
|
-
@env.unextend
|
325
|
-
assert_equal 42, @env[:blah]
|
1624
|
+
def test_yylex_tilde_unary
|
1625
|
+
@lex.lex_state = :expr_fname
|
1626
|
+
util_lex_token "~@", :tTILDE, "~"
|
326
1627
|
end
|
327
1628
|
|
328
|
-
def
|
329
|
-
|
330
|
-
|
1629
|
+
def test_yylex_uminus
|
1630
|
+
util_lex_token("-blah",
|
1631
|
+
:tUMINUS, "-",
|
1632
|
+
:tIDENTIFIER, "blah")
|
1633
|
+
end
|
331
1634
|
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
assert_equal expected1, @env.dynamic
|
1635
|
+
def test_yylex_underscore
|
1636
|
+
util_lex_token("_var", :tIDENTIFIER, "_var")
|
1637
|
+
end
|
336
1638
|
|
337
|
-
|
338
|
-
|
1639
|
+
def test_yylex_underscore_end
|
1640
|
+
@lex.src = "__END__\n"
|
1641
|
+
deny @lex.advance
|
1642
|
+
end
|
339
1643
|
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
1644
|
+
def test_yylex_uplus
|
1645
|
+
util_lex_token("+blah",
|
1646
|
+
:tUPLUS, "+",
|
1647
|
+
:tIDENTIFIER, "blah")
|
1648
|
+
end
|
345
1649
|
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
1650
|
+
def test_zbug_float_in_decl
|
1651
|
+
util_lex_token("def initialize(u = ",
|
1652
|
+
:kDEF, "def",
|
1653
|
+
:tIDENTIFIER, "initialize",
|
1654
|
+
:tLPAREN2, "(",
|
1655
|
+
:tIDENTIFIER, "u",
|
1656
|
+
:tEQL, "=")
|
1657
|
+
|
1658
|
+
assert_equal :expr_beg, @lex.lex_state
|
1659
|
+
|
1660
|
+
util_lex_token("0.0, s = 0.0",
|
1661
|
+
:tFLOAT, 0.0,
|
1662
|
+
:tCOMMA, ',',
|
1663
|
+
:tIDENTIFIER, "s",
|
1664
|
+
:tEQL, "=",
|
1665
|
+
:tFLOAT, 0.0)
|
350
1666
|
end
|
351
1667
|
|
352
|
-
def
|
353
|
-
|
1668
|
+
def test_zbug_id_equals
|
1669
|
+
util_lex_token("a =",
|
1670
|
+
:tIDENTIFIER, "a",
|
1671
|
+
:tEQL, "=")
|
1672
|
+
|
1673
|
+
assert_equal :expr_beg, @lex.lex_state
|
1674
|
+
|
1675
|
+
util_lex_token("0.0",
|
1676
|
+
:tFLOAT, 0.0)
|
1677
|
+
end
|
354
1678
|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
1679
|
+
def test_zbug_no_spaces_in_decl
|
1680
|
+
util_lex_token("def initialize(u=",
|
1681
|
+
:kDEF, "def",
|
1682
|
+
:tIDENTIFIER, "initialize",
|
1683
|
+
:tLPAREN2, "(",
|
1684
|
+
:tIDENTIFIER, "u",
|
1685
|
+
:tEQL, "=")
|
1686
|
+
|
1687
|
+
assert_equal :expr_beg, @lex.lex_state
|
1688
|
+
|
1689
|
+
util_lex_token("0.0,s=0.0",
|
1690
|
+
:tFLOAT, 0.0,
|
1691
|
+
:tCOMMA, ",",
|
1692
|
+
:tIDENTIFIER, "s",
|
1693
|
+
:tEQL, "=",
|
1694
|
+
:tFLOAT, 0.0)
|
359
1695
|
end
|
360
1696
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
1697
|
+
############################################################
|
1698
|
+
|
1699
|
+
def util_bad_token s, *args
|
1700
|
+
assert_raises SyntaxError do
|
1701
|
+
util_lex_token s, *args
|
1702
|
+
end
|
1703
|
+
end
|
365
1704
|
|
366
|
-
|
367
|
-
|
368
|
-
assert_equal expected, @
|
1705
|
+
def util_escape expected, input
|
1706
|
+
@lex.src = input
|
1707
|
+
assert_equal expected, @lex.read_escape
|
369
1708
|
end
|
370
1709
|
|
371
|
-
def
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
assert_equal false, @env.dynamic?
|
1710
|
+
def util_escape_bad input
|
1711
|
+
@lex.src = input
|
1712
|
+
assert_raises SyntaxError do
|
1713
|
+
@lex.read_escape
|
1714
|
+
end
|
377
1715
|
end
|
378
1716
|
|
379
|
-
def
|
380
|
-
|
381
|
-
|
382
|
-
|
1717
|
+
def util_lex_fname name, type, end_state = :expr_arg
|
1718
|
+
@lex.lex_state = :expr_fname # can only set via parser's defs
|
1719
|
+
|
1720
|
+
util_lex_token("def #{name} ", :kDEF, "def", type, name)
|
383
1721
|
|
384
|
-
@
|
385
|
-
|
386
|
-
assert_equal expected1, @env.all
|
1722
|
+
assert_equal end_state, @lex.lex_state
|
1723
|
+
end
|
387
1724
|
|
388
|
-
|
389
|
-
@
|
390
|
-
assert_equal expected2, @env.all
|
1725
|
+
def util_lex_token input, *args
|
1726
|
+
@lex.src = input
|
391
1727
|
|
392
|
-
|
393
|
-
|
1728
|
+
until args.empty? do
|
1729
|
+
token = args.shift
|
1730
|
+
value = args.shift
|
1731
|
+
assert @lex.advance, "no more tokens"
|
1732
|
+
assert_equal [token, value], [@lex.token, @lex.yacc_value]
|
1733
|
+
end
|
394
1734
|
|
395
|
-
@
|
396
|
-
assert_equal expected0, @env.all
|
1735
|
+
deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
|
397
1736
|
end
|
398
1737
|
end
|
1738
|
+
|