parser 2.7.1.1 → 3.0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -0
  3. data/lib/parser/all.rb +2 -0
  4. data/lib/parser/ast/processor.rb +5 -0
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +263 -23
  7. data/lib/parser/context.rb +5 -0
  8. data/lib/parser/current.rb +24 -6
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/diagnostic.rb +1 -1
  11. data/lib/parser/diagnostic/engine.rb +1 -2
  12. data/lib/parser/lexer.rb +887 -803
  13. data/lib/parser/macruby.rb +2214 -2189
  14. data/lib/parser/max_numparam_stack.rb +13 -5
  15. data/lib/parser/messages.rb +18 -0
  16. data/lib/parser/meta.rb +6 -5
  17. data/lib/parser/ruby18.rb +9 -3
  18. data/lib/parser/ruby19.rb +2297 -2289
  19. data/lib/parser/ruby20.rb +2413 -2397
  20. data/lib/parser/ruby21.rb +2419 -2411
  21. data/lib/parser/ruby22.rb +2468 -2460
  22. data/lib/parser/ruby23.rb +2452 -2452
  23. data/lib/parser/ruby24.rb +2435 -2430
  24. data/lib/parser/ruby25.rb +2220 -2214
  25. data/lib/parser/ruby26.rb +2220 -2214
  26. data/lib/parser/ruby27.rb +3715 -3615
  27. data/lib/parser/ruby28.rb +8047 -0
  28. data/lib/parser/ruby30.rb +8060 -0
  29. data/lib/parser/ruby31.rb +8226 -0
  30. data/lib/parser/rubymotion.rb +2190 -2182
  31. data/lib/parser/runner.rb +31 -2
  32. data/lib/parser/runner/ruby_rewrite.rb +2 -2
  33. data/lib/parser/source/buffer.rb +53 -28
  34. data/lib/parser/source/comment.rb +14 -1
  35. data/lib/parser/source/comment/associator.rb +31 -8
  36. data/lib/parser/source/map/method_definition.rb +25 -0
  37. data/lib/parser/source/range.rb +10 -3
  38. data/lib/parser/source/tree_rewriter.rb +100 -10
  39. data/lib/parser/source/tree_rewriter/action.rb +114 -21
  40. data/lib/parser/static_environment.rb +4 -0
  41. data/lib/parser/tree_rewriter.rb +1 -2
  42. data/lib/parser/variables_stack.rb +4 -0
  43. data/lib/parser/version.rb +1 -1
  44. data/parser.gemspec +3 -18
  45. metadata +17 -98
  46. data/.gitignore +0 -33
  47. data/.travis.yml +0 -42
  48. data/.yardopts +0 -21
  49. data/CHANGELOG.md +0 -1075
  50. data/CONTRIBUTING.md +0 -17
  51. data/Gemfile +0 -10
  52. data/README.md +0 -309
  53. data/Rakefile +0 -166
  54. data/ci/run_rubocop_specs +0 -14
  55. data/doc/AST_FORMAT.md +0 -2180
  56. data/doc/CUSTOMIZATION.md +0 -37
  57. data/doc/INTERNALS.md +0 -21
  58. data/doc/css/.gitkeep +0 -0
  59. data/doc/css/common.css +0 -68
  60. data/lib/parser/lexer.rl +0 -2536
  61. data/lib/parser/macruby.y +0 -2198
  62. data/lib/parser/ruby18.y +0 -1934
  63. data/lib/parser/ruby19.y +0 -2175
  64. data/lib/parser/ruby20.y +0 -2353
  65. data/lib/parser/ruby21.y +0 -2357
  66. data/lib/parser/ruby22.y +0 -2364
  67. data/lib/parser/ruby23.y +0 -2370
  68. data/lib/parser/ruby24.y +0 -2408
  69. data/lib/parser/ruby25.y +0 -2405
  70. data/lib/parser/ruby26.y +0 -2413
  71. data/lib/parser/ruby27.y +0 -2941
  72. data/lib/parser/rubymotion.y +0 -2182
  73. data/test/bug_163/fixtures/input.rb +0 -5
  74. data/test/bug_163/fixtures/output.rb +0 -5
  75. data/test/bug_163/rewriter.rb +0 -20
  76. data/test/helper.rb +0 -60
  77. data/test/parse_helper.rb +0 -319
  78. data/test/racc_coverage_helper.rb +0 -133
  79. data/test/test_base.rb +0 -31
  80. data/test/test_current.rb +0 -29
  81. data/test/test_diagnostic.rb +0 -96
  82. data/test/test_diagnostic_engine.rb +0 -62
  83. data/test/test_encoding.rb +0 -99
  84. data/test/test_lexer.rb +0 -3608
  85. data/test/test_lexer_stack_state.rb +0 -78
  86. data/test/test_parse_helper.rb +0 -80
  87. data/test/test_parser.rb +0 -9430
  88. data/test/test_runner_parse.rb +0 -35
  89. data/test/test_runner_rewrite.rb +0 -47
  90. data/test/test_source_buffer.rb +0 -162
  91. data/test/test_source_comment.rb +0 -36
  92. data/test/test_source_comment_associator.rb +0 -367
  93. data/test/test_source_map.rb +0 -15
  94. data/test/test_source_range.rb +0 -187
  95. data/test/test_source_rewriter.rb +0 -541
  96. data/test/test_source_rewriter_action.rb +0 -46
  97. data/test/test_source_tree_rewriter.rb +0 -253
  98. data/test/test_static_environment.rb +0 -45
  99. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  100. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  101. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
data/test/test_base.rb DELETED
@@ -1,31 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
- require 'parser/current'
5
-
6
- class TestBase < Minitest::Test
7
- include AST::Sexp
8
-
9
- def test_parse
10
- ast = Parser::CurrentRuby.parse('1')
11
- assert_equal s(:int, 1), ast
12
- end
13
-
14
- def test_parse_with_comments
15
- ast, comments = Parser::CurrentRuby.parse_with_comments('1 # foo')
16
- assert_equal s(:int, 1), ast
17
- assert_equal 1, comments.size
18
- assert_equal '# foo', comments.first.text
19
- end
20
-
21
- def test_loc_to_node
22
- ast = Parser::CurrentRuby.parse('1')
23
- assert_equal ast.loc.node, ast
24
- end
25
-
26
- def test_loc_dup
27
- ast = Parser::CurrentRuby.parse('1')
28
- assert_nil ast.loc.dup.node
29
- Parser::AST::Node.new(:root, [], :location => ast.loc)
30
- end
31
- end
data/test/test_current.rb DELETED
@@ -1,29 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
- require 'parser/current'
5
-
6
- class TestCurrent < Minitest::Test
7
- def test_current
8
- case RUBY_VERSION
9
- when '2.0.0'
10
- assert_equal Parser::Ruby20, Parser::CurrentRuby
11
- when /^2\.1\.\d+/
12
- assert_equal Parser::Ruby21, Parser::CurrentRuby
13
- when /^2\.2\.\d+/
14
- assert_equal Parser::Ruby22, Parser::CurrentRuby
15
- when /^2\.3\.\d+/
16
- assert_equal Parser::Ruby23, Parser::CurrentRuby
17
- when /^2\.4\.\d+/
18
- assert_equal Parser::Ruby24, Parser::CurrentRuby
19
- when /^2\.5\.\d+/
20
- assert_equal Parser::Ruby25, Parser::CurrentRuby
21
- when /^2\.6\.\d+/
22
- assert_equal Parser::Ruby26, Parser::CurrentRuby
23
- when /^2\.7\.\d+/
24
- assert_equal Parser::Ruby27, Parser::CurrentRuby
25
- else
26
- flunk "Update test_current for #{RUBY_VERSION}"
27
- end
28
- end
29
- end
@@ -1,96 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
-
5
- class TestDiagnostic < Minitest::Test
6
- def setup
7
- @buffer = Parser::Source::Buffer.new('(string)')
8
- @buffer.source = 'if (this is some bad code + bugs)'
9
-
10
- @range1 = Parser::Source::Range.new(@buffer, 0, 2) # if
11
- @range2 = Parser::Source::Range.new(@buffer, 4, 8) # this
12
- end
13
-
14
- def test_verifies_levels
15
- error = assert_raises ArgumentError do
16
- Parser::Diagnostic.new(:foobar, :escape_eof, {}, @range1)
17
- end
18
-
19
- assert_match /level/, error.message
20
- end
21
-
22
- def test_freezes
23
- string = 'foo'.dup
24
- highlights = [@range2]
25
-
26
- diag = Parser::Diagnostic.new(:error, :escape_eof, @range1, highlights)
27
- assert diag.frozen?
28
- assert diag.arguments.frozen?
29
- assert diag.highlights.frozen?
30
-
31
- refute string.frozen?
32
- refute highlights.frozen?
33
- end
34
-
35
- def test_render
36
- location = Parser::Source::Range.new(@buffer, 26, 27)
37
-
38
- highlights = [
39
- Parser::Source::Range.new(@buffer, 21, 25),
40
- Parser::Source::Range.new(@buffer, 28, 32)
41
- ]
42
-
43
- diag = Parser::Diagnostic.new(:error, :unexpected, { :character => '+' },
44
- location, highlights)
45
- assert_equal([
46
- "(string):1:27: error: unexpected `+'",
47
- '(string):1: if (this is some bad code + bugs)',
48
- '(string):1: ~~~~ ^ ~~~~ '
49
- ], diag.render)
50
- end
51
-
52
- def test_multiline_render
53
- @buffer = Parser::Source::Buffer.new('(string)')
54
- @buffer.source = "abc abc abc\ndef def def\nghi ghi ghi\n"
55
-
56
- location = Parser::Source::Range.new(@buffer, 4, 27)
57
-
58
- highlights = [
59
- Parser::Source::Range.new(@buffer, 0, 3),
60
- Parser::Source::Range.new(@buffer, 28, 31)
61
- ]
62
-
63
- diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'ghi' },
64
- location, highlights)
65
-
66
- assert_equal([
67
- "(string):1:5-3:3: error: unexpected token ghi",
68
- '(string):1: abc abc abc',
69
- '(string):1: ~~~ ^~~~~~~...',
70
- '(string):3: ghi ghi ghi',
71
- '(string):3: ~~~ ~~~ '
72
- ], diag.render)
73
- end
74
-
75
- def test_bug_error_on_newline
76
- # regression test; see GitHub issue 273
77
- source = <<-CODE
78
- {
79
- foo: ->() # I forgot my brace
80
- }
81
- }
82
- CODE
83
- @buffer = Parser::Source::Buffer.new('(string)')
84
- @buffer.source = source
85
-
86
- location = Parser::Source::Range.new(@buffer, 33, 34)
87
- diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'tNL' },
88
- location)
89
-
90
- assert_equal([
91
- '(string):2:32: error: unexpected token tNL',
92
- '(string):2: foo: ->() # I forgot my brace',
93
- '(string):2: ^'
94
- ], diag.render)
95
- end
96
- end
@@ -1,62 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
-
5
- class TestDiagnosticEngine < Minitest::Test
6
- def setup
7
- @buffer = Parser::Source::Buffer.new('(source)')
8
- @buffer.source = 'foobar'
9
-
10
- @engine = Parser::Diagnostic::Engine.new
11
-
12
- @queue = []
13
- @engine.consumer = lambda { |diag| @queue << diag }
14
- end
15
-
16
- def test_process_warnings
17
- warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
18
- @engine.process(warn)
19
-
20
- assert_equal [warn], @queue
21
- end
22
-
23
- def test_ignore_warnings
24
- @engine.ignore_warnings = true
25
-
26
- warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
27
- @engine.process(warn)
28
-
29
- assert_equal [], @queue
30
- end
31
-
32
- def test_all_errors_are_fatal
33
- @engine.all_errors_are_fatal = true
34
-
35
- error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
36
-
37
- err = assert_raises Parser::SyntaxError do
38
- @engine.process(error)
39
- end
40
-
41
- assert_equal error, err.diagnostic
42
-
43
- assert_equal [error], @queue
44
- end
45
-
46
- def test_all_errors_are_collected
47
- error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
48
- @engine.process(error)
49
-
50
- assert_equal [error], @queue
51
- end
52
-
53
- def test_fatal_error
54
- fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, @buffer, 1..2)
55
-
56
- assert_raises Parser::SyntaxError do
57
- @engine.process(fatal)
58
- end
59
-
60
- assert_equal [fatal], @queue
61
- end
62
- end
@@ -1,99 +0,0 @@
1
- # encoding: binary
2
- # frozen_string_literal: true
3
-
4
- require 'helper'
5
-
6
- class TestEncoding < Minitest::Test
7
- include AST::Sexp
8
-
9
- def recognize(string)
10
- Parser::Source::Buffer.recognize_encoding(string)
11
- end
12
-
13
- require 'parser/all'
14
-
15
- def test_default
16
- assert_nil recognize('foobar')
17
- end
18
-
19
- def test_bom
20
- assert_equal Encoding::UTF_8, recognize("\xef\xbb\xbf\nfoobar")
21
- assert_equal Encoding::UTF_8, recognize("\xef\xbb\xbf# coding:koi8-r\nfoobar")
22
- end
23
-
24
- def test_magic_comment
25
- assert_equal Encoding::KOI8_R, recognize("# coding:koi8-r\nfoobar")
26
- end
27
-
28
- def test_shebang
29
- assert_equal Encoding::KOI8_R, recognize("#!/bin/foo\n# coding:koi8-r\nfoobar")
30
- assert_nil recognize("#!/bin/foo\n")
31
- end
32
-
33
- def test_case
34
- assert_equal Encoding::KOI8_R, recognize("# coding:KoI8-r\nfoobar")
35
- end
36
-
37
- def test_space
38
- assert_equal Encoding::KOI8_R, recognize("# coding : koi8-r\nfoobar")
39
- end
40
-
41
- def test_empty
42
- assert_nil recognize('')
43
- end
44
-
45
- def test_no_comment
46
- assert_nil recognize(%{require 'cane/encoding_aware_iterator'})
47
- end
48
-
49
- def test_adjacent
50
- assert_nil recognize('# codingkoi8-r')
51
- assert_nil recognize('# coding koi8-r')
52
- end
53
-
54
- def test_utf8_mac
55
- assert_equal Encoding::UTF8_MAC, recognize('# coding: utf8-mac')
56
- end
57
-
58
- def test_suffix
59
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-dos')
60
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-unix')
61
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-mac')
62
-
63
- assert_raises(ArgumentError) do
64
- assert_nil recognize('# coding: utf-8-dicks')
65
- end
66
- end
67
-
68
- def test_parse_18_invalid_enc
69
- ast = Parser::Ruby18.parse("# encoding:feynman-diagram\n1")
70
- assert_equal ast, s(:int, 1)
71
- end
72
-
73
- def test_parse_19_invalid_enc
74
- assert_raises(ArgumentError) do
75
- Parser::Ruby19.parse("# encoding:feynman-diagram\n1")
76
- end
77
- end
78
-
79
- def test_ending_comment
80
- assert_nil recognize('foo # coding: koi8-r')
81
- end
82
-
83
- def test_wrong_prefix
84
- assert_nil recognize('# decoding: koi8-r')
85
- end
86
-
87
- def test_no_spaces
88
- assert_equal Encoding::KOI8_R, recognize('#encoding:koi8-r')
89
- assert_equal Encoding::KOI8_R, recognize('#coding:koi8-r')
90
- end
91
-
92
- def test_underscore_and_star_characters
93
- assert_equal Encoding::KOI8_R, recognize('# -*- encoding: koi8-r -*-')
94
- end
95
-
96
- def test_garbage_around_encoding_comment
97
- assert_equal Encoding::KOI8_R, recognize('# 1$# -*- &)* encoding: koi8-r 1$# -*- &)*')
98
- end
99
- end
data/test/test_lexer.rb DELETED
@@ -1,3608 +0,0 @@
1
- # encoding: ascii-8bit
2
- # frozen_string_literal: true
3
-
4
- require 'helper'
5
- require 'complex'
6
-
7
- class TestLexer < Minitest::Test
8
- def setup_lexer(version)
9
- @lex = Parser::Lexer.new(version)
10
-
11
- @lex.comments = []
12
- @lex.diagnostics = Parser::Diagnostic::Engine.new
13
- @lex.diagnostics.all_errors_are_fatal = true
14
- # @lex.diagnostics.consumer = lambda { |diag| $stderr.puts "", diag.render }
15
- end
16
-
17
- def setup
18
- setup_lexer 18
19
- end
20
-
21
- #
22
- # Tools
23
- #
24
-
25
- def utf(str)
26
- str.dup.force_encoding(Encoding::UTF_8)
27
- end
28
-
29
- #
30
- # Additional matchers
31
- #
32
-
33
- def refute_scanned(s, *args)
34
- assert_raises Parser::SyntaxError do
35
- assert_scanned(s, *args)
36
- end
37
- end
38
-
39
- def assert_escape(expected, input)
40
- source_buffer = Parser::Source::Buffer.new('(assert_escape)')
41
-
42
- source_buffer.source = "\"\\#{input}\"".encode(input.encoding)
43
-
44
- @lex.reset
45
- @lex.source_buffer = source_buffer
46
-
47
- lex_token, (lex_value, *) = @lex.advance
48
-
49
- lex_value.force_encoding(Encoding::BINARY)
50
-
51
- assert_equal [:tSTRING, expected],
52
- [lex_token, lex_value],
53
- source_buffer.source
54
- end
55
-
56
- def refute_escape(input)
57
- err = assert_raises Parser::SyntaxError do
58
- @lex.state = :expr_beg
59
- assert_scanned "%Q[\\#{input}]"
60
- end
61
- assert_equal :fatal, err.diagnostic.level
62
- end
63
-
64
- def assert_lex_fname(name, type, range)
65
- begin_pos, end_pos = range
66
- assert_scanned("def #{name} ",
67
- :kDEF, 'def', [0, 3],
68
- type, name, [begin_pos + 4, end_pos + 4])
69
-
70
- assert_equal :expr_endfn, @lex.state
71
- end
72
-
73
- def assert_scanned(input, *args)
74
- source_buffer = Parser::Source::Buffer.new('(assert_scanned)')
75
- source_buffer.source = input
76
-
77
- @lex.reset(false)
78
- @lex.source_buffer = source_buffer
79
-
80
- until args.empty? do
81
- token, value, (begin_pos, end_pos) = args.shift(3)
82
-
83
- lex_token, (lex_value, lex_range) = @lex.advance
84
- assert lex_token, 'no more tokens'
85
- assert_operator [lex_token, lex_value], :eql?, [token, value], input
86
- assert_equal begin_pos, lex_range.begin_pos
87
- assert_equal end_pos, lex_range.end_pos
88
- end
89
-
90
- lex_token, (lex_value, *) = @lex.advance
91
- refute lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
92
- end
93
-
94
- #
95
- # Tests
96
- #
97
-
98
- def test_read_escape
99
- assert_escape "\\", "\\"
100
- assert_escape "\n", "n"
101
- assert_escape "\t", "t"
102
- assert_escape "\r", "r"
103
- assert_escape "\f", "f"
104
- assert_escape "\13", "v"
105
- assert_escape "\0", "0"
106
- assert_escape "\07", "a"
107
- assert_escape "\007", "a"
108
- assert_escape "\033", "e"
109
- assert_escape "\377", "377"
110
- assert_escape "\377", "xff"
111
- assert_escape "\010", "b"
112
- assert_escape " ", "s"
113
- assert_escape "q", "q" # plain vanilla escape
114
- end
115
-
116
- def test_read_escape_c
117
- assert_escape "\030", "C-x"
118
- assert_escape "\030", "cx"
119
- assert_escape "\230", 'C-\M-x'
120
- assert_escape "\230", 'c\M-x'
121
-
122
- assert_escape "\177", "C-?"
123
- assert_escape "\177", "c?"
124
- assert_escape "\r", "cM"
125
- end
126
-
127
- def test_read_escape_m
128
- assert_escape "\370", "M-x"
129
- assert_escape "\230", 'M-\C-x'
130
- assert_escape "\230", 'M-\cx'
131
- end
132
-
133
- def test_read_escape_errors
134
- refute_escape ""
135
-
136
- refute_escape "M"
137
- refute_escape "M-"
138
- refute_escape "Mx"
139
-
140
- refute_escape "Cx"
141
- refute_escape "C"
142
- refute_escape "C-"
143
-
144
- refute_escape "c"
145
-
146
- refute_escape "x"
147
- end
148
-
149
- def test_read_escape_unicode__19
150
- assert_escape "\x09", 'u{9}'
151
- assert_escape "\x31", 'u{31}'
152
- assert_escape "\x09\x01", 'u{9 1}'
153
-
154
- assert_escape "\xc4\xa3", utf('u0123')
155
- assert_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", utf('u{123 f0 B123}')
156
- end
157
-
158
- def test_read_escape_unicode_bad__19
159
- refute_escape 'u123'
160
- refute_escape 'u{}'
161
- refute_escape 'u{123 f0h}'
162
- refute_escape 'u{123 f0'
163
- end
164
-
165
- def test_read_escape_whitespaces__27
166
- setup_lexer 27
167
-
168
- [ *(0..8), *(14..31) ].each do |code|
169
- @lex.reset
170
- refute_scanned "\"\\C-" + code.chr + "\""
171
-
172
- @lex.reset
173
- refute_scanned "\"\\M-" + code.chr + "\""
174
-
175
- @lex.reset
176
- refute_scanned "\"\\C-\\M-" + code.chr + "\""
177
-
178
- @lex.reset
179
- refute_scanned "\"\\M-\\C-" + code.chr + "\""
180
- end
181
- end
182
-
183
- def test_ambiguous_uminus
184
- assert_scanned("m -3",
185
- :tIDENTIFIER, "m", [0, 1],
186
- :tUNARY_NUM, "-", [2, 3],
187
- :tINTEGER, 3, [3, 4])
188
- end
189
-
190
- def test_ambiguous_uplus
191
- assert_scanned("m +3",
192
- :tIDENTIFIER, "m", [0, 1],
193
- :tUNARY_NUM, "+", [2, 3],
194
- :tINTEGER, 3, [3, 4])
195
- end
196
-
197
- def test_and
198
- assert_scanned "&", :tAMPER, "&", [0, 1]
199
- end
200
-
201
- def test_and2
202
- @lex.state = :expr_end
203
-
204
- assert_scanned "&&", :tANDOP, "&&", [0, 2]
205
- end
206
-
207
- def test_and2_equals
208
- @lex.state = :expr_end
209
-
210
- assert_scanned "&&=", :tOP_ASGN, "&&", [0, 3]
211
- end
212
-
213
- def test_and_arg
214
- @lex.state = :expr_arg
215
-
216
- assert_scanned(" &y",
217
- :tAMPER, "&", [1, 2],
218
- :tIDENTIFIER, "y", [2, 3])
219
- end
220
-
221
- def test_and_equals
222
- @lex.state = :expr_end
223
-
224
- assert_scanned "&=", :tOP_ASGN, "&", [0, 2]
225
- end
226
-
227
- def test_and_expr
228
- @lex.state = :expr_arg
229
-
230
- assert_scanned("x & y",
231
- :tIDENTIFIER, "x", [0, 1],
232
- :tAMPER2, "&", [2, 3],
233
- :tIDENTIFIER, "y", [4, 5])
234
- end
235
-
236
- def test_and_meth
237
- assert_lex_fname "&", :tAMPER2, [0, 1]
238
- end
239
-
240
- def test_and_dot_arg
241
- @lex.state = :expr_arg
242
-
243
- assert_scanned "&.", :tANDDOT, "&.", [0, 2]
244
- end
245
-
246
- def test_and_dot_cmdarg
247
- @lex.state = :expr_cmdarg
248
-
249
- assert_scanned "&.", :tANDDOT, "&.", [0, 2]
250
- end
251
-
252
- def test_assoc
253
- assert_scanned "=>", :tASSOC, "=>", [0, 2]
254
- end
255
-
256
- def test_label__18
257
- assert_scanned("{a:b",
258
- :tLBRACE, "{", [0, 1],
259
- :tIDENTIFIER, "a", [1, 2],
260
- :tSYMBOL, "b", [2, 4])
261
- end
262
-
263
- def test_label_in_params__18
264
- assert_scanned("foo(a:b",
265
- :tIDENTIFIER, "foo", [0, 3],
266
- :tLPAREN2, "(", [3, 4],
267
- :tIDENTIFIER, "a", [4, 5],
268
- :tSYMBOL, "b", [5, 7])
269
- end
270
-
271
- def test_label__19
272
- setup_lexer 19
273
-
274
- assert_scanned("{a:b",
275
- :tLBRACE, "{", [0, 1],
276
- :tLABEL, "a", [1, 3],
277
- :tIDENTIFIER, "b", [3, 4])
278
- end
279
-
280
- def test_label_in_params__19
281
- setup_lexer 19
282
-
283
- assert_scanned("foo(a:b",
284
- :tIDENTIFIER, "foo", [0, 3],
285
- :tLPAREN2, "(", [3, 4],
286
- :tLABEL, "a", [4, 6],
287
- :tIDENTIFIER, "b", [6, 7])
288
- end
289
-
290
- def test_label_fid__19
291
- setup_lexer 19
292
-
293
- assert_scanned("{a?:true",
294
- :tLBRACE, '{', [0, 1],
295
- :tLABEL, 'a?', [1, 4],
296
- :kTRUE, 'true', [4, 8])
297
- end
298
-
299
- def test_label__22
300
- setup_lexer 22
301
-
302
- assert_scanned("{'a':",
303
- :tLBRACE, '{', [0, 1],
304
- :tSTRING_BEG, "'", [1, 2],
305
- :tSTRING_CONTENT, 'a', [2, 3],
306
- :tLABEL_END, "'", [3, 5])
307
- end
308
-
309
- def test_label_nested__22
310
- setup_lexer 22
311
-
312
- assert_scanned("{'a\":':",
313
- :tLBRACE, '{', [0, 1],
314
- :tSTRING_BEG, "'", [1, 2],
315
- :tSTRING_CONTENT, 'a":', [2, 5],
316
- :tLABEL_END, "'", [5, 7])
317
- end
318
-
319
- def test_label_colon2__22
320
- setup_lexer 22
321
-
322
- assert_scanned("{'a'::",
323
- :tLBRACE, '{', [0, 1],
324
- :tSTRING, "a", [1, 4],
325
- :tCOLON2, '::', [4, 6])
326
- end
327
-
328
- def test_pct_string_colon__22
329
- setup_lexer 22
330
-
331
- assert_scanned("{%'a':",
332
- :tLBRACE, '{', [0, 1],
333
- :tSTRING_BEG, "%'", [1, 3],
334
- :tSTRING_CONTENT, 'a', [3, 4],
335
- :tSTRING_END, "'", [4, 5],
336
- :tCOLON, ':', [5, 6])
337
- end
338
-
339
- def test_command_start__19
340
- setup_lexer 19
341
-
342
- %w[case elsif for in until when while
343
- if unless and or].each do |keyword|
344
- token = "k#{keyword.upcase}".to_sym
345
-
346
- @lex.reset
347
- assert_scanned("#{keyword} a:b",
348
- token, keyword, [0, keyword.length],
349
- :tIDENTIFIER, "a", [keyword.length + 1, keyword.length + 2],
350
- :tSYMBOL, "b", [keyword.length + 2, keyword.length + 4])
351
- end
352
- end
353
-
354
- def test_mod_not_command_start__19
355
- setup_lexer 19
356
-
357
- %w[if unless while until rescue].each do |keyword|
358
- token = "k#{keyword.upcase}_MOD".to_sym
359
-
360
- @lex.state = :expr_end
361
- assert_scanned("#{keyword} a:b",
362
- token, keyword, [0, keyword.length],
363
- :tLABEL, "a", [keyword.length + 1, keyword.length + 3],
364
- :tIDENTIFIER, "b", [keyword.length + 3, keyword.length + 4])
365
- end
366
- end
367
-
368
- def test_back_ref
369
- assert_scanned("[$&, $`, $', $+]",
370
- :tLBRACK, "[", [0, 1],
371
- :tBACK_REF, "$&", [1, 3], :tCOMMA, ",", [3, 4],
372
- :tBACK_REF, "$`", [5, 7], :tCOMMA, ",", [7, 8],
373
- :tBACK_REF, "$'", [9, 11], :tCOMMA, ",", [11, 12],
374
- :tBACK_REF, "$+", [13, 15],
375
- :tRBRACK, "]", [15, 16])
376
- end
377
-
378
- def test_backslash
379
- assert_scanned("1 \\\n+ 2",
380
- :tINTEGER, 1, [0, 1],
381
- :tPLUS, "+", [4, 5],
382
- :tINTEGER, 2, [6, 7])
383
- end
384
-
385
- def test_backslash_bad
386
- refute_scanned("1 \\ + 2",
387
- :tINTEGER, 1, [0, 1])
388
- end
389
-
390
- def test_backtick
391
- assert_scanned("`ls`",
392
- :tXSTRING_BEG, "`", [0, 1],
393
- :tSTRING_CONTENT, "ls", [1, 3],
394
- :tSTRING_END, "`", [3, 4])
395
- end
396
-
397
- def test_backtick_cmdarg
398
- @lex.state = :expr_dot
399
- assert_scanned("\n`", :tBACK_REF2, "`", [1, 2]) # \n ensures expr_cmd
400
-
401
- assert_equal :expr_arg, @lex.state
402
- end
403
-
404
- def test_backtick_dot
405
- @lex.state = :expr_dot
406
- assert_scanned("a.`(3)",
407
- :tIDENTIFIER, "a", [0, 1],
408
- :tDOT, ".", [1, 2],
409
- :tBACK_REF2, "`", [2, 3],
410
- :tLPAREN2, "(", [3, 4],
411
- :tINTEGER, 3, [4, 5],
412
- :tRPAREN, ")", [5, 6])
413
- end
414
-
415
- def test_backtick_method
416
- @lex.state = :expr_fname
417
- assert_scanned("`", :tBACK_REF2, "`", [0, 1])
418
- assert_equal :expr_endfn, @lex.state
419
- end
420
-
421
- def test_bad_char
422
- refute_scanned(" \010 ")
423
- end
424
-
425
- def test_bang
426
- assert_scanned "!", :tBANG, "!", [0, 1]
427
- end
428
-
429
- def test_bang_equals
430
- assert_scanned "!=", :tNEQ, "!=", [0, 2]
431
- end
432
-
433
- def test_bang_tilde
434
- assert_scanned "!~", :tNMATCH, "!~", [0, 2]
435
- end
436
-
437
- def test_def_ubang
438
- setup_lexer(20)
439
-
440
- @lex.state = :expr_fname
441
- assert_scanned '!@', :tBANG, '!@', [0, 2]
442
- end
443
-
444
- def test_carat
445
- assert_scanned "^", :tCARET, "^", [0, 1]
446
- end
447
-
448
- def test_carat_equals
449
- assert_scanned "^=", :tOP_ASGN, "^", [0, 2]
450
- end
451
-
452
- def test_colon2
453
- assert_scanned("A::B",
454
- :tCONSTANT, "A", [0, 1],
455
- :tCOLON2, "::", [1, 3],
456
- :tCONSTANT, "B", [3, 4])
457
-
458
- @lex.state = :expr_arg
459
- assert_scanned("::Array",
460
- :tCOLON2, "::", [0, 2],
461
- :tCONSTANT, "Array", [2, 7])
462
- end
463
-
464
- def test_colon3
465
- assert_scanned("::Array",
466
- :tCOLON3, "::", [0, 2],
467
- :tCONSTANT, "Array", [2, 7])
468
-
469
- @lex.state = :expr_arg
470
- assert_scanned(" ::Array",
471
- :tCOLON3, "::", [1, 3],
472
- :tCONSTANT, "Array", [3, 8])
473
- end
474
-
475
- def test_comma
476
- assert_scanned ",", :tCOMMA, ",", [0, 1]
477
- end
478
-
479
- def test_comment
480
- [26, 27].each do |version|
481
- setup_lexer(version)
482
-
483
- assert_scanned("1 # one\n# two\n2",
484
- :tINTEGER, 1, [0, 1],
485
- :tNL, nil, [7, 8],
486
- :tINTEGER, 2, [14, 15])
487
-
488
- assert_equal 2, @lex.comments.length
489
- assert_equal '# one', @lex.comments[0].text
490
- assert_equal '# two', @lex.comments[1].text
491
- end
492
- end
493
-
494
- def test_comment_expr_beg
495
- assert_scanned("{#1\n}",
496
- :tLBRACE, "{", [0, 1],
497
- :tRCURLY, "}", [4, 5])
498
- end
499
-
500
- def test_comment_begin
501
- assert_scanned("=begin\nblah\nblah\n=end\n42",
502
- :tINTEGER, 42, [22, 24])
503
- assert_equal 1, @lex.comments.length
504
- assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments[0].text
505
- end
506
-
507
- def test_comment_begin_bad
508
- refute_scanned("=begin\nblah\nblah\n")
509
- end
510
-
511
- def test_comment_begin_not_comment
512
- assert_scanned("beginfoo = 5\np x \\\n=beginfoo",
513
- :tIDENTIFIER, "beginfoo", [0, 8],
514
- :tEQL, "=", [9, 10],
515
- :tINTEGER, 5, [11, 12],
516
- :tNL, nil, [12, 13],
517
- :tIDENTIFIER, "p", [13, 14],
518
- :tIDENTIFIER, "x", [15, 16],
519
- :tEQL, "=", [19, 20],
520
- :tIDENTIFIER, "beginfoo", [20, 28])
521
- end
522
-
523
- def test_comment_begin_space
524
- assert_scanned("=begin blah\nblah\n=end\n")
525
-
526
- assert_equal 1, @lex.comments.length
527
- assert_equal "=begin blah\nblah\n=end\n", @lex.comments[0].text
528
- end
529
-
530
- def test_comment_end_space_and_text
531
- assert_scanned("=begin blah\nblah\n=end blab\n")
532
-
533
- assert_equal 1, @lex.comments.length
534
- assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments[0].text
535
- end
536
-
537
- def test_comment_eos
538
- assert_scanned("# comment")
539
- end
540
-
541
- def test_constant
542
- assert_scanned("ArgumentError",
543
- :tCONSTANT, "ArgumentError", [0, 13])
544
- end
545
-
546
- def test_constant_semi
547
- assert_scanned("ArgumentError;",
548
- :tCONSTANT, "ArgumentError", [0, 13],
549
- :tSEMI, ";", [13, 14])
550
- end
551
-
552
- def test_cvar
553
- assert_scanned "@@blah", :tCVAR, "@@blah", [0, 6]
554
- end
555
-
556
- def test_cvar_bad
557
- refute_scanned "@@1"
558
- end
559
-
560
- def test_div
561
- assert_scanned("a / 2",
562
- :tIDENTIFIER, "a", [0, 1],
563
- :tDIVIDE, "/", [2, 3],
564
- :tINTEGER, 2, [4, 5])
565
- end
566
-
567
- def test_div_equals
568
- assert_scanned("a /= 2",
569
- :tIDENTIFIER, "a", [0, 1],
570
- :tOP_ASGN, "/", [2, 4],
571
- :tINTEGER, 2, [5, 6])
572
- end
573
-
574
- def test_do
575
- assert_scanned("x do 42 end",
576
- :tIDENTIFIER, "x", [0, 1],
577
- :kDO, "do", [2, 4],
578
- :tINTEGER, 42, [5, 7],
579
- :kEND, "end", [8, 11])
580
- end
581
-
582
- def test_do_block
583
- @lex.state = :expr_endarg
584
-
585
- assert_scanned("do 42 end",
586
- :kDO_BLOCK, "do", [0, 2],
587
- :tINTEGER, 42, [3, 5],
588
- :kEND, "end", [6, 9])
589
- end
590
-
591
- def test_do_cond
592
- @lex.cond.push true
593
-
594
- assert_scanned("x do 42 end",
595
- :tIDENTIFIER, "x", [0, 1],
596
- :kDO_COND, "do", [2, 4],
597
- :tINTEGER, 42, [5, 7],
598
- :kEND, "end", [8, 11])
599
- end
600
-
601
- def test_dot
602
- assert_scanned ".", :tDOT, ".", [0, 1]
603
- end
604
-
605
- def test_dot2
606
- assert_scanned "..", :tDOT2, "..", [0, 2]
607
- end
608
-
609
- def test_dot3
610
- assert_scanned "...", :tDOT3, "...", [0, 3]
611
- end
612
-
613
- def test_equals
614
- assert_scanned "=", :tEQL, "=", [0, 1]
615
- end
616
-
617
- def test_equals2
618
- assert_scanned "==", :tEQ, "==", [0, 2]
619
- end
620
-
621
- def test_equals3
622
- assert_scanned "===", :tEQQ, "===", [0, 3]
623
- end
624
-
625
- def test_equals_tilde
626
- assert_scanned "=~", :tMATCH, "=~", [0, 2]
627
- end
628
-
629
- def test_float
630
- assert_scanned "1.0", :tFLOAT, 1.0, [0, 3]
631
- end
632
-
633
- def test_float_bad_no_underscores
634
- refute_scanned "1__0.0"
635
- end
636
-
637
- def test_float_bad_no_zero_leading
638
- refute_scanned ".0"
639
- end
640
-
641
- def test_float_bad_trailing_underscore
642
- refute_scanned "123_.0"
643
- end
644
-
645
- def test_float_call
646
- assert_scanned("1.0.to_s",
647
- :tFLOAT, 1.0, [0, 3],
648
- :tDOT, ".", [3, 4],
649
- :tIDENTIFIER, "to_s", [4, 8])
650
- end
651
-
652
- def test_float_dot_E
653
- assert_scanned "1.0E10", :tFLOAT, 1.0e10, [0, 6]
654
- end
655
-
656
- def test_float_dot_E_neg
657
- assert_scanned("-1.0E10",
658
- :tUNARY_NUM, "-", [0, 1],
659
- :tFLOAT, 1.0e10, [1, 7])
660
- end
661
-
662
- def test_float_dot_E_pos
663
- assert_scanned("+1.0E10",
664
- :tUNARY_NUM, "+", [0, 1],
665
- :tFLOAT, 1.0e10, [1, 7])
666
- end
667
-
668
- def test_float_dot_e
669
- assert_scanned "1.0e10", :tFLOAT, 1.0e10, [0, 6]
670
- end
671
-
672
- def test_float_dot_e_neg
673
- assert_scanned("-1.0e10",
674
- :tUNARY_NUM, "-", [0, 1],
675
- :tFLOAT, 1.0e10, [1, 7])
676
- end
677
-
678
- def test_float_dot_e_pos
679
- assert_scanned("+1.0e10",
680
- :tUNARY_NUM, "+", [0, 1],
681
- :tFLOAT, 1.0e10, [1, 7])
682
- end
683
-
684
- def test_float_e
685
- assert_scanned "1e10", :tFLOAT, 1e10, [0, 4]
686
- end
687
-
688
- def test_float_e_bad_trailing_underscore
689
- refute_scanned "123_e10"
690
- end
691
-
692
- def test_float_e_minus
693
- assert_scanned "1e-10", :tFLOAT, 1e-10, [0, 5]
694
- end
695
-
696
- def test_float_e_neg
697
- assert_scanned("-1e10",
698
- :tUNARY_NUM, "-", [0, 1],
699
- :tFLOAT, 1e10, [1, 5])
700
- end
701
-
702
- def test_float_e_neg_minus
703
- assert_scanned("-1e-10",
704
- :tUNARY_NUM, "-", [0, 1],
705
- :tFLOAT, 1e-10, [1, 6])
706
- end
707
-
708
- def test_float_e_neg_plus
709
- assert_scanned("-1e+10",
710
- :tUNARY_NUM, "-", [0, 1],
711
- :tFLOAT, 1e10, [1, 6])
712
- end
713
-
714
- def test_float_e_pos
715
- assert_scanned("+1e10",
716
- :tUNARY_NUM, "+", [0, 1],
717
- :tFLOAT, 1e10, [1, 5])
718
- end
719
-
720
- def test_float_e_pos_minus
721
- assert_scanned("+1e-10",
722
- :tUNARY_NUM, "+", [0, 1],
723
- :tFLOAT, 1e-10, [1, 6])
724
- end
725
-
726
- def test_float_e_pos_plus
727
- assert_scanned("+1e+10",
728
- :tUNARY_NUM, "+", [0, 1],
729
- :tFLOAT, 1e10, [1, 6])
730
- end
731
-
732
- def test_float_e_plus
733
- assert_scanned "1e+10", :tFLOAT, 1e10, [0, 5]
734
- end
735
-
736
- def test_float_e_zero
737
- assert_scanned "0e0", :tFLOAT, 0e0, [0, 3]
738
- end
739
-
740
- def test_float_e_nothing
741
- [18, 19, 20].each do |version|
742
- setup_lexer version
743
-
744
- refute_scanned "1end"
745
- refute_scanned "1.1end"
746
- end
747
-
748
- setup_lexer 21
749
-
750
- assert_scanned("1end",
751
- :tINTEGER, 1, [0, 1],
752
- :kEND, 'end', [1, 4])
753
- assert_scanned("1.1end",
754
- :tFLOAT, 1.1, [0, 3],
755
- :kEND, 'end', [3, 6])
756
- end
757
-
758
- def test_float_neg
759
- assert_scanned("-1.0",
760
- :tUNARY_NUM, "-", [0, 1],
761
- :tFLOAT, 1.0, [1, 4])
762
- end
763
-
764
- def test_float_pos
765
- assert_scanned("+1.0",
766
- :tUNARY_NUM, "+", [0, 1],
767
- :tFLOAT, 1.0, [1, 4])
768
- end
769
-
770
- def test_ge
771
- assert_scanned("a >= 2",
772
- :tIDENTIFIER, "a", [0, 1],
773
- :tGEQ, ">=", [2, 4],
774
- :tINTEGER, 2, [5, 6])
775
- end
776
-
777
- def test_global
778
- assert_scanned("$blah", :tGVAR, "$blah", [0, 5])
779
- end
780
-
781
- def test_global_backref
782
- assert_scanned("$`", :tBACK_REF, "$`", [0, 2])
783
- end
784
-
785
- # This was removed in 2.1.
786
- # def test_global_dash_nothing
787
- # assert_scanned("$- ", :tGVAR, "$-")
788
- # end
789
-
790
- def test_global_dash_something
791
- assert_scanned("$-x", :tGVAR, "$-x", [0, 3])
792
- end
793
-
794
- def test_global_number
795
- assert_scanned("$10", :tNTH_REF, 10, [0, 3])
796
- end
797
-
798
- def test_global_other
799
- assert_scanned("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
800
- :tLBRACK, "[", [0, 1],
801
- :tGVAR, "$~", [1, 3], :tCOMMA, ",", [3, 4],
802
- :tGVAR, "$*", [5, 7], :tCOMMA, ",", [7, 8],
803
- :tGVAR, "$$", [9, 11], :tCOMMA, ",", [11, 12],
804
- :tGVAR, "$\?", [13, 15], :tCOMMA, ",", [15, 16],
805
- :tGVAR, "$!", [17, 19], :tCOMMA, ",", [19, 20],
806
- :tGVAR, "$@", [21, 23], :tCOMMA, ",", [23, 24],
807
- :tGVAR, "$/", [25, 27], :tCOMMA, ",", [27, 28],
808
- :tGVAR, "$\\", [29, 31], :tCOMMA, ",", [31, 32],
809
- :tGVAR, "$;", [33, 35], :tCOMMA, ",", [35, 36],
810
- :tGVAR, "$,", [37, 39], :tCOMMA, ",", [39, 40],
811
- :tGVAR, "$.", [41, 43], :tCOMMA, ",", [43, 44],
812
- :tGVAR, "$=", [45, 47], :tCOMMA, ",", [47, 48],
813
- :tGVAR, "$:", [49, 51], :tCOMMA, ",", [51, 52],
814
- :tGVAR, "$<", [53, 55], :tCOMMA, ",", [55, 56],
815
- :tGVAR, "$>", [57, 59], :tCOMMA, ",", [59, 60],
816
- :tGVAR, "$\"", [61, 63],
817
- :tRBRACK, "]", [63, 64])
818
- end
819
-
820
- def test_global_underscore
821
- assert_scanned("$_",
822
- :tGVAR, "$_", [0, 2])
823
- end
824
-
825
- def test_global_weird
826
- assert_scanned("$__blah",
827
- :tGVAR, "$__blah", [0, 7])
828
- end
829
-
830
- def test_global_zero
831
- assert_scanned("$0", :tGVAR, "$0", [0, 2])
832
- end
833
-
834
- def test_gt
835
- assert_scanned("a > 2",
836
- :tIDENTIFIER, "a", [0, 1],
837
- :tGT, ">", [2, 3],
838
- :tINTEGER, 2, [4, 5])
839
- end
840
-
841
- def test_heredoc_backtick
842
- assert_scanned("a = <<`EOF`\n blah blah\nEOF\n",
843
- :tIDENTIFIER, "a", [0, 1],
844
- :tEQL, "=", [2, 3],
845
- :tXSTRING_BEG, "<<`", [4, 11],
846
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
847
- :tSTRING_END, "EOF", [24, 27],
848
- :tNL, nil, [11, 12])
849
- end
850
-
851
- def test_heredoc_double
852
- assert_scanned("a = <<\"EOF\"\n blah blah\nEOF\n",
853
- :tIDENTIFIER, "a", [0, 1],
854
- :tEQL, "=", [2, 3],
855
- :tSTRING_BEG, "<<\"", [4, 11],
856
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
857
- :tSTRING_END, "EOF", [24, 27],
858
- :tNL, nil, [11, 12])
859
- end
860
-
861
- def test_heredoc_double_dash
862
- assert_scanned("a = <<-\"EOF\"\n blah blah\n EOF\n",
863
- :tIDENTIFIER, "a", [0, 1],
864
- :tEQL, "=", [2, 3],
865
- :tSTRING_BEG, "<<\"", [4, 12],
866
- :tSTRING_CONTENT, " blah blah\n", [13, 25],
867
- :tSTRING_END, "EOF", [25, 30],
868
- :tNL, nil, [12, 13])
869
- end
870
-
871
- def test_heredoc_double_eos
872
- refute_scanned("a = <<\"EOF\"\nblah",
873
- :tIDENTIFIER, "a", [0, 1],
874
- :tEQL, "=", [2, 3],
875
- :tSTRING_BEG, "<<\"", [4, 7])
876
- end
877
-
878
- def test_heredoc_double_eos_nl
879
- refute_scanned("a = <<\"EOF\"\nblah\n",
880
- :tIDENTIFIER, "a", [0, 1],
881
- :tEQL, "=", [2, 3],
882
- :tSTRING_BEG, "<<\"", [4, 7])
883
- end
884
-
885
- def test_heredoc_double_interp
886
- assert_scanned("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
887
- :tIDENTIFIER, "a", [0, 1],
888
- :tEQL, "=", [2, 3],
889
- :tSTRING_BEG, "<<\"", [4, 11],
890
- :tSTRING_CONTENT, "#x a ", [12, 17],
891
- :tSTRING_DVAR, nil, [17, 18],
892
- :tIVAR, "@a", [18, 20],
893
- :tSTRING_CONTENT, " b ", [20, 23],
894
- :tSTRING_DVAR, nil, [23, 24],
895
- :tGVAR, "$b", [24, 26],
896
- :tSTRING_CONTENT, " c ", [26, 29],
897
- :tSTRING_DBEG, '#{', [29, 31],
898
- :tINTEGER, 3, [31, 32],
899
- :tRCURLY, "}", [32, 33],
900
- :tSTRING_CONTENT, " \n", [33, 35],
901
- :tSTRING_END, "EOF", [35, 38],
902
- :tNL, nil, [11, 12])
903
- end
904
-
905
- def test_heredoc_empty
906
- assert_scanned("<<\"\"\n\#{x}\nblah2\n\n",
907
- :tSTRING_BEG, "<<\"", [0, 4],
908
- :tSTRING_DBEG, "\#{", [5, 7],
909
- :tIDENTIFIER, "x", [7, 8],
910
- :tRCURLY, "}", [8, 9],
911
- :tSTRING_CONTENT, "\n", [9, 10],
912
- :tSTRING_CONTENT, "blah2\n", [10, 16],
913
- :tSTRING_END, "", [16, 16],
914
- :tNL, nil, [4, 5])
915
- end
916
-
917
- def test_heredoc_none
918
- assert_scanned("a = <<EOF\nblah\nblah\nEOF",
919
- :tIDENTIFIER, "a", [0, 1],
920
- :tEQL, "=", [2, 3],
921
- :tSTRING_BEG, "<<\"", [4, 9],
922
- :tSTRING_CONTENT, "blah\n", [10, 15],
923
- :tSTRING_CONTENT, "blah\n", [15, 20],
924
- :tSTRING_END, "EOF", [20, 23],
925
- :tNL, nil, [9, 10])
926
- end
927
-
928
- def test_heredoc_none_dash
929
- assert_scanned("a = <<-EOF\nblah\nblah\n EOF",
930
- :tIDENTIFIER, "a", [0, 1],
931
- :tEQL, "=", [2, 3],
932
- :tSTRING_BEG, "<<\"", [4, 10],
933
- :tSTRING_CONTENT, "blah\n", [11, 16],
934
- :tSTRING_CONTENT, "blah\n", [16, 21],
935
- :tSTRING_END, "EOF", [21, 26],
936
- :tNL, nil, [10, 11])
937
- end
938
-
939
- def test_heredoc_single
940
- assert_scanned("a = <<'EOF'\n blah blah\nEOF\n",
941
- :tIDENTIFIER, "a", [0, 1],
942
- :tEQL, "=", [2, 3],
943
- :tSTRING_BEG, "<<'", [4, 11],
944
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
945
- :tSTRING_END, "EOF", [24, 27],
946
- :tNL, nil, [11, 12])
947
- end
948
-
949
- def test_heredoc_single_bad_eos_body
950
- refute_scanned("a = <<'EOF'\nblah",
951
- :tIDENTIFIER, "a", [0, 1],
952
- :tEQL, "=", [2, 3],
953
- :tSTRING_BEG, "'", [6, 7])
954
- end
955
-
956
- def test_heredoc_single_dash
957
- assert_scanned("a = <<-'EOF'\n blah blah\n EOF\n",
958
- :tIDENTIFIER, "a", [0, 1],
959
- :tEQL, "=", [2, 3],
960
- :tSTRING_BEG, "<<'", [4, 12],
961
- :tSTRING_CONTENT, " blah blah\n", [13, 25],
962
- :tSTRING_END, "EOF", [25, 30],
963
- :tNL, nil, [12, 13])
964
- end
965
-
966
- def test_heredoc_one_character
967
- assert_scanned("a = <<E\nABCDEF\nE\n",
968
- :tIDENTIFIER, "a", [0, 1],
969
- :tEQL, "=", [2, 3],
970
- :tSTRING_BEG, "<<\"", [4, 7],
971
- :tSTRING_CONTENT, "ABCDEF\n", [8, 15],
972
- :tSTRING_END, "E", [15, 16],
973
- :tNL, nil, [7, 8])
974
- end
975
-
976
- def test_heredoc_cr
977
- assert_scanned("a = <<E\r\r\nABCDEF\r\r\nE\r\r\r\n",
978
- :tIDENTIFIER, "a", [0, 1],
979
- :tEQL, "=", [2, 3],
980
- :tSTRING_BEG, "<<\"", [4, 7],
981
- :tSTRING_CONTENT, "ABCDEF\r\n", [9, 17],
982
- :tSTRING_END, "E", [17, 20],
983
- :tNL, nil, [8, 9])
984
- end
985
-
986
- def test_heredoc_with_identifier_ending_newline__19
987
- setup_lexer 19
988
- refute_scanned "<<\"EOS\n\"\n123\nEOS\n"
989
- end
990
-
991
- def test_heredoc_with_identifier_ending_newline__24
992
- setup_lexer 24
993
-
994
- assert_scanned("a = <<\"EOS\n\"\nABCDEF\nEOS\n",
995
- :tIDENTIFIER, "a", [0, 1],
996
- :tEQL, "=", [2, 3],
997
- :tSTRING_BEG, "<<\"", [4, 12],
998
- :tSTRING_CONTENT, "ABCDEF\n", [13, 20],
999
- :tSTRING_END, "EOS", [20, 23],
1000
- :tNL, nil, [12, 13])
1001
- end
1002
-
1003
- def test_heredoc_with_identifier_containing_newline_inside__19
1004
- setup_lexer 19
1005
- refute_scanned "<<\"EOS\nEOS\"\n123\nEOS\n"
1006
- end
1007
-
1008
- def test_heredoc_with_identifier_containing_newline_inside__24
1009
- setup_lexer 24
1010
-
1011
- refute_scanned "<<\"EOS\nEOS\"\n123\nEOS\n"
1012
- end
1013
-
1014
- def test_identifier
1015
- assert_scanned("identifier",
1016
- :tIDENTIFIER, "identifier", [0, 10])
1017
- end
1018
-
1019
- def test_identifier_bang
1020
- assert_scanned("identifier!",
1021
- :tFID, "identifier!", [0, 11])
1022
-
1023
- assert_scanned("identifier!=",
1024
- :tIDENTIFIER, "identifier", [0, 10],
1025
- :tNEQ, "!=", [10, 12])
1026
- end
1027
-
1028
- def test_identifier_eh
1029
- setup_lexer 19
1030
-
1031
- assert_scanned("identifier?",
1032
- :tFID, "identifier?", [0, 11])
1033
-
1034
- assert_scanned("identifier?=",
1035
- :tIDENTIFIER, "identifier", [0, 10],
1036
- :tCHARACTER, "=", [10, 12])
1037
- end
1038
-
1039
- def test_identifier_cmp
1040
- assert_lex_fname "<=>", :tCMP, [0, 3]
1041
- end
1042
-
1043
- def test_identifier_def
1044
- assert_lex_fname "identifier", :tIDENTIFIER, [0, 10]
1045
- end
1046
-
1047
- def test_identifier_equals_arrow
1048
- assert_scanned(":blah==>",
1049
- :tSYMBOL, "blah=", [0, 6],
1050
- :tASSOC, "=>", [6, 8])
1051
- end
1052
-
1053
- def test_identifier_equals3
1054
- assert_scanned(":a===b",
1055
- :tSYMBOL, "a", [0, 2],
1056
- :tEQQ, "===", [2, 5],
1057
- :tIDENTIFIER, "b", [5, 6])
1058
- end
1059
-
1060
- def test_identifier_equals_equals_arrow
1061
- assert_scanned(":a==>b",
1062
- :tSYMBOL, "a=", [0, 3],
1063
- :tASSOC, "=>", [3, 5],
1064
- :tIDENTIFIER, "b", [5, 6])
1065
- end
1066
-
1067
- def test_identifier_equals_caret
1068
- assert_lex_fname "^", :tCARET, [0, 1]
1069
- end
1070
-
1071
- def test_identifier_equals_def
1072
- assert_lex_fname "identifier=", :tIDENTIFIER, [0, 11]
1073
- end
1074
-
1075
- def test_identifier_equals_def2
1076
- assert_lex_fname "==", :tEQ, [0, 2]
1077
- end
1078
-
1079
- def test_identifier_equals_expr
1080
- @lex.state = :expr_dot
1081
- assert_scanned("y = arg",
1082
- :tIDENTIFIER, "y", [0, 1],
1083
- :tEQL, "=", [2, 3],
1084
- :tIDENTIFIER, "arg", [4, 7])
1085
-
1086
- assert_equal :expr_arg, @lex.state
1087
- end
1088
-
1089
- def test_identifier_equals_or
1090
- assert_lex_fname "|", :tPIPE, [0, 1]
1091
- end
1092
-
1093
- def test_identifier_equals_slash
1094
- assert_lex_fname "/", :tDIVIDE, [0, 1]
1095
- end
1096
-
1097
- def test_identifier_equals_tilde
1098
- @lex.state = :expr_fname
1099
- assert_scanned("identifier=~",
1100
- :tIDENTIFIER, "identifier=", [0, 11],
1101
- :tTILDE, "~", [11, 12])
1102
- end
1103
-
1104
- def test_identifier_gt
1105
- assert_lex_fname ">", :tGT, [0, 1]
1106
- end
1107
-
1108
- def test_identifier_le
1109
- assert_lex_fname "<=", :tLEQ, [0, 2]
1110
- end
1111
-
1112
- def test_identifier_lt
1113
- assert_lex_fname "<", :tLT, [0, 1]
1114
- end
1115
-
1116
- def test_identifier_tilde
1117
- assert_lex_fname "~", :tTILDE, [0, 1]
1118
- end
1119
-
1120
- def test_identifier_defined?
1121
- assert_lex_fname "defined?", :kDEFINED, [0, 8]
1122
- end
1123
-
1124
- def test_index
1125
- assert_lex_fname "[]", :tAREF, [0, 2]
1126
- end
1127
-
1128
- def test_index_equals
1129
- assert_lex_fname "[]=", :tASET, [0, 3]
1130
- end
1131
-
1132
- def test_integer
1133
- assert_scanned "42", :tINTEGER, 42, [0, 2]
1134
- end
1135
-
1136
- def test_integer_bin
1137
- assert_scanned "0b101010", :tINTEGER, 42, [0, 8]
1138
- end
1139
-
1140
- def test_integer_bin_bad_none
1141
- refute_scanned "0b "
1142
- end
1143
-
1144
- def test_integer_bin_bad_underscores
1145
- refute_scanned "0b10__01"
1146
- end
1147
-
1148
- def test_integer_dec
1149
- assert_scanned "42", :tINTEGER, 42, [0, 2]
1150
- end
1151
-
1152
- def test_integer_dec_bad_underscores
1153
- refute_scanned "42__24"
1154
- end
1155
-
1156
- def test_integer_dec_d
1157
- assert_scanned "0d42", :tINTEGER, 42, [0, 4]
1158
- end
1159
-
1160
- def test_integer_dec_d_bad_none
1161
- refute_scanned "0d"
1162
- end
1163
-
1164
- def test_integer_dec_d_bad_underscores
1165
- refute_scanned "0d42__24"
1166
- end
1167
-
1168
- def test_question_eh_a__18
1169
- setup_lexer 18
1170
-
1171
- assert_scanned "?a", :tINTEGER, 97, [0, 2]
1172
- end
1173
-
1174
- def test_question_eh_a__19
1175
- setup_lexer 19
1176
-
1177
- assert_scanned '?a', :tCHARACTER, "a", [0, 2]
1178
- end
1179
-
1180
- def test_question_eh_escape_M_escape_C__18
1181
- setup_lexer 18
1182
-
1183
- assert_scanned '?\M-\C-a', :tINTEGER, 129, [0, 8]
1184
- end
1185
-
1186
- def test_question_eh_escape_M_escape_C__19
1187
- setup_lexer 19
1188
-
1189
- assert_scanned '?\M-\C-a', :tCHARACTER, "\M-\C-a", [0, 8]
1190
- end
1191
-
1192
- def test_question_eh_escape_u_1_digit
1193
- setup_lexer 19
1194
-
1195
- refute_scanned '?\\u1'
1196
- end
1197
-
1198
- def test_question_eh_escape_u_2_digits
1199
- setup_lexer 19
1200
-
1201
- refute_scanned '?\\u12'
1202
- end
1203
-
1204
- def test_question_eh_escape_u_3_digits
1205
- setup_lexer 19
1206
-
1207
- refute_scanned '?\\u123'
1208
- end
1209
-
1210
- def test_question_eh_escape_u_4_digits
1211
- setup_lexer 19
1212
- assert_scanned '?\\u0001', :tCHARACTER, "\u0001", [0, 7]
1213
- end
1214
-
1215
- def test_question_eh_single_unicode_point
1216
- setup_lexer 19
1217
- assert_scanned '?\\u{123}', :tCHARACTER, "\u0123", [0, 8]
1218
-
1219
- setup_lexer 19
1220
- assert_scanned '?\\u{a}', :tCHARACTER, "\n", [0, 6]
1221
- end
1222
-
1223
- def test_question_eh_multiple_unicode_points
1224
- setup_lexer 19
1225
- refute_scanned '?\\u{1 2 3}'
1226
-
1227
- setup_lexer 19
1228
- refute_scanned '?\\u{a b}'
1229
- end
1230
-
1231
- def test_question_eh_escape_u_unclosed_bracket
1232
- setup_lexer 19
1233
-
1234
- refute_scanned '?\\u{123'
1235
- end
1236
-
1237
- def test_question_eh_escape_space_around_unicode_point__19
1238
- setup_lexer 19
1239
- refute_scanned '"\\u{1 }"'
1240
-
1241
- setup_lexer 19
1242
- refute_scanned '"\\u{ 1}"'
1243
-
1244
- setup_lexer 19
1245
- refute_scanned '"\\u{ 1 }"'
1246
-
1247
- setup_lexer 19
1248
- refute_scanned '"\\u{1 2 }"'
1249
-
1250
- setup_lexer 19
1251
- refute_scanned '"\\u{ 1 2}"'
1252
-
1253
- setup_lexer 19
1254
- refute_scanned '"\\u{1 2}"'
1255
- end
1256
-
1257
- def test_question_eh_escape_space_around_unicode_point__24
1258
- setup_lexer 24
1259
- assert_scanned '"\\u{ 1}"', :tSTRING, "\u0001", [0, 8]
1260
-
1261
- setup_lexer 24
1262
- assert_scanned '"\\u{1 }"', :tSTRING, "\u0001", [0, 8]
1263
-
1264
- setup_lexer 24
1265
- assert_scanned '"\\u{ 1 }"', :tSTRING, "\u0001", [0, 9]
1266
-
1267
- setup_lexer 24
1268
- assert_scanned '"\\u{1 2 }"', :tSTRING, "\u0001\u0002", [0, 10]
1269
-
1270
- setup_lexer 24
1271
- assert_scanned '"\\u{ 1 2}"', :tSTRING, "\u0001\u0002", [0, 10]
1272
-
1273
- setup_lexer 24
1274
- assert_scanned '"\\u{1 2}"', :tSTRING, "\u0001\u0002", [0, 10]
1275
- end
1276
-
1277
- def test_integer_hex
1278
- assert_scanned "0x2a", :tINTEGER, 42, [0, 4]
1279
- end
1280
-
1281
- def test_integer_hex_bad_none
1282
- refute_scanned "0x "
1283
- end
1284
-
1285
- def test_integer_hex_bad_underscores
1286
- refute_scanned "0xab__cd"
1287
- end
1288
-
1289
- def test_integer_oct
1290
- assert_scanned "052", :tINTEGER, 42, [0, 3]
1291
- end
1292
-
1293
- def test_integer_oct_bad_range
1294
- refute_scanned "08"
1295
- end
1296
-
1297
- def test_integer_oct_bad_underscores
1298
- refute_scanned "01__23"
1299
- end
1300
-
1301
- def test_integer_oct_O
1302
- assert_scanned "0O52", :tINTEGER, 42, [0, 4]
1303
- end
1304
-
1305
- def test_integer_oct_O_bad_range
1306
- refute_scanned "0O1238"
1307
- end
1308
-
1309
- def test_integer_oct_O_bad_underscores
1310
- refute_scanned "0O1__23"
1311
- end
1312
-
1313
- def test_integer_oct_O_not_bad_none
1314
- assert_scanned "0O ", :tINTEGER, 0, [0, 2]
1315
- end
1316
-
1317
- def test_integer_oct_o
1318
- assert_scanned "0o52", :tINTEGER, 42, [0, 4]
1319
- end
1320
-
1321
- def test_integer_oct_o_bad_range
1322
- refute_scanned "0o1283"
1323
- end
1324
-
1325
- def test_integer_oct_o_bad_underscores
1326
- refute_scanned "0o1__23"
1327
- end
1328
-
1329
- def test_integer_oct_o_not_bad_none
1330
- assert_scanned "0o ", :tINTEGER, 0, [0, 2]
1331
- end
1332
-
1333
- def test_integer_trailing
1334
- assert_scanned("1.to_s",
1335
- :tINTEGER, 1, [0, 1],
1336
- :tDOT, '.', [1, 2],
1337
- :tIDENTIFIER, 'to_s', [2, 6])
1338
- end
1339
-
1340
- def test_integer_underscore
1341
- assert_scanned "4_2", :tINTEGER, 42, [0, 3]
1342
- end
1343
-
1344
- def test_integer_underscore_bad
1345
- refute_scanned "4__2"
1346
- end
1347
-
1348
- def test_integer_zero
1349
- assert_scanned "0", :tINTEGER, 0, [0, 1]
1350
- end
1351
-
1352
- def test_ivar
1353
- assert_scanned "@blah", :tIVAR, "@blah", [0, 5]
1354
- end
1355
-
1356
- def test_ivar_bad
1357
- refute_scanned "@1"
1358
- end
1359
-
1360
- def test_ivar_bad_0_length
1361
- refute_scanned "1+@\n", :tINTEGER, 1, [0, 1], :tPLUS, "+", [1, 2]
1362
- end
1363
-
1364
- def test_keyword_expr
1365
- @lex.state = :expr_endarg
1366
-
1367
- assert_scanned "if", :kIF_MOD, "if", [0, 2]
1368
-
1369
- assert_equal :expr_beg, @lex.state
1370
- end
1371
-
1372
- def test_lt
1373
- assert_scanned "<", :tLT, "<", [0, 1]
1374
- end
1375
-
1376
- def test_lt2
1377
- assert_scanned("a <\< b",
1378
- :tIDENTIFIER, "a", [0, 1],
1379
- :tLSHFT, "<\<", [2, 4],
1380
- :tIDENTIFIER, "b", [5, 6])
1381
-
1382
- end
1383
-
1384
- def test_lt2_equals
1385
- assert_scanned("a <\<= b",
1386
- :tIDENTIFIER, "a", [0, 1],
1387
- :tOP_ASGN, "<\<", [2, 5],
1388
- :tIDENTIFIER, "b", [6, 7])
1389
- end
1390
-
1391
- def test_lt_equals
1392
- assert_scanned "<=", :tLEQ, "<=", [0, 2]
1393
- end
1394
-
1395
- def test_minus
1396
- assert_scanned("1 - 2",
1397
- :tINTEGER, 1, [0, 1],
1398
- :tMINUS, "-", [2, 3],
1399
- :tINTEGER, 2, [4, 5])
1400
- end
1401
-
1402
- def test_minus_equals
1403
- @lex.state = :expr_end
1404
-
1405
- assert_scanned "-=", :tOP_ASGN, "-", [0, 2]
1406
- end
1407
-
1408
- def test_minus_method
1409
- @lex.state = :expr_fname
1410
- assert_scanned "-", :tMINUS, "-", [0, 1]
1411
- end
1412
-
1413
- def test_minus_unary_method
1414
- @lex.state = :expr_fname
1415
- assert_scanned "-@", :tUMINUS, "-@", [0, 2]
1416
- end
1417
-
1418
- def test_minus_unary_number
1419
- assert_scanned("-42",
1420
- :tUNARY_NUM, "-", [0, 1],
1421
- :tINTEGER, 42, [1, 3])
1422
- end
1423
-
1424
- def test_minus_unary_whitespace_number
1425
- assert_scanned("- 42",
1426
- :tUNARY_NUM, "-", [0, 1],
1427
- :tINTEGER, 42, [2, 4])
1428
- end
1429
-
1430
- def test_nth_ref
1431
- assert_scanned('[$1, $2, $3]',
1432
- :tLBRACK, "[", [0, 1],
1433
- :tNTH_REF, 1, [1, 3], :tCOMMA, ",", [3, 4],
1434
- :tNTH_REF, 2, [5, 7], :tCOMMA, ",", [7, 8],
1435
- :tNTH_REF, 3, [9, 11],
1436
- :tRBRACK, "]", [11, 12])
1437
- end
1438
-
1439
- def test_open_bracket
1440
- assert_scanned("(", :tLPAREN, "(", [0, 1])
1441
- end
1442
-
1443
- def test_open_bracket_cmdarg
1444
- assert_scanned("m (", :tIDENTIFIER, "m", [0, 1],
1445
- :tLPAREN_ARG, "(", [2, 3])
1446
- end
1447
-
1448
- def test_open_bracket_exprarg
1449
- assert_scanned("m(", :tIDENTIFIER, "m", [0, 1],
1450
- :tLPAREN2, "(", [1, 2])
1451
- end
1452
-
1453
- def test_open_curly_bracket
1454
- assert_scanned("{",
1455
- :tLBRACE, "{", [0, 1])
1456
- end
1457
-
1458
- def test_open_curly_bracket_arg
1459
- assert_scanned("m { 3 }",
1460
- :tIDENTIFIER, "m", [0, 1],
1461
- :tLCURLY, "{", [2, 3],
1462
- :tINTEGER, 3, [4, 5],
1463
- :tRCURLY, "}", [6, 7])
1464
- end
1465
-
1466
- def test_open_curly_bracket_block
1467
- @lex.state = :expr_endarg # seen m(3)
1468
-
1469
- assert_scanned("{ 4 }",
1470
- :tLBRACE_ARG, "{", [0, 1],
1471
- :tINTEGER, 4, [2, 3],
1472
- :tRCURLY, "}", [4, 5])
1473
- end
1474
-
1475
- def test_open_square_bracket_arg
1476
- assert_scanned("m [ 3 ]",
1477
- :tIDENTIFIER, "m", [0, 1],
1478
- :tLBRACK, "[", [2, 3],
1479
- :tINTEGER, 3, [4, 5],
1480
- :tRBRACK, "]", [6, 7])
1481
- end
1482
-
1483
- def test_open_square_bracket_ary
1484
- assert_scanned("[1, 2, 3]",
1485
- :tLBRACK, "[", [0, 1],
1486
- :tINTEGER, 1, [1, 2],
1487
- :tCOMMA, ",", [2, 3],
1488
- :tINTEGER, 2, [4, 5],
1489
- :tCOMMA, ",", [5, 6],
1490
- :tINTEGER, 3, [7, 8],
1491
- :tRBRACK, "]", [8, 9])
1492
- end
1493
-
1494
- def test_open_square_bracket_meth
1495
- assert_scanned("m[3]",
1496
- :tIDENTIFIER, "m", [0, 1],
1497
- :tLBRACK2, "[", [1, 2],
1498
- :tINTEGER, 3, [2, 3],
1499
- :tRBRACK, "]", [3, 4])
1500
- end
1501
-
1502
- def test_or
1503
- assert_scanned "|", :tPIPE, "|", [0, 1]
1504
- end
1505
-
1506
- def test_or2
1507
- assert_scanned "||", :tOROP, "||", [0, 2]
1508
- end
1509
-
1510
- def test_or2__after_27
1511
- setup_lexer(27)
1512
- assert_scanned("||",
1513
- :tPIPE, "|", [0, 1],
1514
- :tPIPE, "|", [1, 2])
1515
- end
1516
-
1517
- def test_or2_equals
1518
- assert_scanned "||=", :tOP_ASGN, "||", [0, 3]
1519
- end
1520
-
1521
- def test_or_equals
1522
- assert_scanned "|=", :tOP_ASGN, "|", [0, 2]
1523
- end
1524
-
1525
- def test_percent
1526
- assert_scanned("a % 2",
1527
- :tIDENTIFIER, "a", [0, 1],
1528
- :tPERCENT, "%", [2, 3],
1529
- :tINTEGER, 2, [4, 5])
1530
- end
1531
-
1532
- def test_percent_equals
1533
- assert_scanned("a %= 2",
1534
- :tIDENTIFIER, "a", [0, 1],
1535
- :tOP_ASGN, "%", [2, 4],
1536
- :tINTEGER, 2, [5, 6])
1537
- end
1538
-
1539
- def test_plus
1540
- assert_scanned("1 + 1",
1541
- :tINTEGER, 1, [0, 1],
1542
- :tPLUS, "+", [2, 3],
1543
- :tINTEGER, 1, [4, 5])
1544
- end
1545
-
1546
- def test_plus_equals
1547
- @lex.state = :expr_end
1548
-
1549
- assert_scanned "+=", :tOP_ASGN, "+", [0, 2]
1550
- end
1551
-
1552
- def test_plus_method
1553
- @lex.state = :expr_fname
1554
- assert_scanned "+", :tPLUS, "+", [0, 1]
1555
- end
1556
-
1557
- def test_plus_unary_method
1558
- @lex.state = :expr_fname
1559
- assert_scanned "+@", :tUPLUS, "+@", [0, 2]
1560
- end
1561
-
1562
- def test_plus_unary_number
1563
- assert_scanned("+42",
1564
- :tUNARY_NUM, "+", [0, 1],
1565
- :tINTEGER, 42, [1, 3])
1566
- end
1567
-
1568
- def test_plus_unary_whitespace_number
1569
- assert_scanned("+ 42",
1570
- :tUNARY_NUM, "+", [0, 1],
1571
- :tINTEGER, 42, [2, 4])
1572
- end
1573
-
1574
- def test_numbers
1575
- assert_scanned "0b10", :tINTEGER, 2, [0, 4]
1576
- assert_scanned "0B10", :tINTEGER, 2, [0, 4]
1577
-
1578
- assert_scanned "0d10", :tINTEGER, 10, [0, 4]
1579
- assert_scanned "0D10", :tINTEGER, 10, [0, 4]
1580
-
1581
- assert_scanned "0x10", :tINTEGER, 16, [0, 4]
1582
- assert_scanned "0X10", :tINTEGER, 16, [0, 4]
1583
-
1584
- assert_scanned "0o10", :tINTEGER, 8, [0, 4]
1585
- assert_scanned "0O10", :tINTEGER, 8, [0, 4]
1586
- assert_scanned "0o", :tINTEGER, 0, [0, 2]
1587
- assert_scanned "0O", :tINTEGER, 0, [0, 2]
1588
-
1589
- assert_scanned "0o", :tINTEGER, 0, [0, 2]
1590
- assert_scanned "0O", :tINTEGER, 0, [0, 2]
1591
-
1592
- assert_scanned "0777_333", :tINTEGER, 261851, [0, 8]
1593
-
1594
- assert_scanned "0", :tINTEGER, 0, [0, 1]
1595
-
1596
- refute_scanned "0x"
1597
- refute_scanned "0X"
1598
- refute_scanned "0b"
1599
- refute_scanned "0B"
1600
- refute_scanned "0d"
1601
- refute_scanned "0D"
1602
-
1603
- refute_scanned "08"
1604
- refute_scanned "09"
1605
- refute_scanned "0o8"
1606
- refute_scanned "0o9"
1607
- refute_scanned "0O8"
1608
- refute_scanned "0O9"
1609
-
1610
- refute_scanned "1_e1"
1611
- refute_scanned "1_.1"
1612
- refute_scanned "1__1"
1613
-
1614
- refute_scanned "1end"
1615
- refute_scanned "1.1end"
1616
- end
1617
-
1618
- def test_question__18
1619
- setup_lexer 18
1620
-
1621
- assert_scanned "?*", :tINTEGER, 42, [0, 2]
1622
- end
1623
-
1624
- def test_question__19
1625
- setup_lexer 19
1626
-
1627
- assert_scanned "?*", :tCHARACTER, "*", [0, 2]
1628
- end
1629
-
1630
- def test_question_bad_eos
1631
- refute_scanned "?"
1632
- end
1633
-
1634
- def test_question_bad_ws
1635
- assert_scanned "? ", :tEH, "?", [0, 1]
1636
- assert_scanned "?\n", :tEH, "?", [0, 1]
1637
- assert_scanned "?\t", :tEH, "?", [0, 1]
1638
- assert_scanned "?\v", :tEH, "?", [0, 1]
1639
- assert_scanned "?\r", :tEH, "?", [0, 1]
1640
- assert_scanned "?\f", :tEH, "?", [0, 1]
1641
- end
1642
-
1643
- def test_question_ws_backslashed__18
1644
- setup_lexer 18
1645
-
1646
- @lex.state = :expr_beg
1647
- assert_scanned "?\\ ", :tINTEGER, 32, [0, 3]
1648
- @lex.state = :expr_beg
1649
- assert_scanned "?\\n", :tINTEGER, 10, [0, 3]
1650
- @lex.state = :expr_beg
1651
- assert_scanned "?\\t", :tINTEGER, 9, [0, 3]
1652
- @lex.state = :expr_beg
1653
- assert_scanned "?\\v", :tINTEGER, 11, [0, 3]
1654
- @lex.state = :expr_beg
1655
- assert_scanned "?\\r", :tINTEGER, 13, [0, 3]
1656
- @lex.state = :expr_beg
1657
- assert_scanned "?\\f", :tINTEGER, 12, [0, 3]
1658
- end
1659
-
1660
- def test_question_ws_backslashed__19
1661
- setup_lexer 19
1662
-
1663
- @lex.state = :expr_beg
1664
- assert_scanned "?\\ ", :tCHARACTER, " ", [0, 3]
1665
- @lex.state = :expr_beg
1666
- assert_scanned "?\\n", :tCHARACTER, "\n", [0, 3]
1667
- @lex.state = :expr_beg
1668
- assert_scanned "?\\t", :tCHARACTER, "\t", [0, 3]
1669
- @lex.state = :expr_beg
1670
- assert_scanned "?\\v", :tCHARACTER, "\v", [0, 3]
1671
- @lex.state = :expr_beg
1672
- assert_scanned "?\\r", :tCHARACTER, "\r", [0, 3]
1673
- @lex.state = :expr_beg
1674
- assert_scanned "?\\f", :tCHARACTER, "\f", [0, 3]
1675
- end
1676
-
1677
- def test_rbracket
1678
- assert_scanned "]", :tRBRACK, "]", [0, 1]
1679
- end
1680
-
1681
- def test_rcurly
1682
- assert_scanned "}", :tRCURLY, "}", [0, 1]
1683
- end
1684
-
1685
- def test_regexp
1686
- assert_scanned("/regexp/",
1687
- :tREGEXP_BEG, "/", [0, 1],
1688
- :tSTRING_CONTENT, "regexp", [1, 7],
1689
- :tSTRING_END, "/", [7, 8],
1690
- :tREGEXP_OPT, "", [8, 8])
1691
- end
1692
-
1693
- def test_regexp_ambiguous
1694
- assert_scanned("method /regexp/",
1695
- :tIDENTIFIER, "method", [0, 6],
1696
- :tREGEXP_BEG, "/", [7, 8],
1697
- :tSTRING_CONTENT, "regexp", [8, 14],
1698
- :tSTRING_END, "/", [14, 15],
1699
- :tREGEXP_OPT, "", [15, 15])
1700
- end
1701
-
1702
- def test_regexp_bad
1703
- refute_scanned("/.*/xyz",
1704
- :tREGEXP_BEG, "/", [0, 1],
1705
- :tSTRING_CONTENT, ".*", [1, 3],
1706
- :tSTRING_END, "/", [3, 4])
1707
- end
1708
-
1709
- def test_regexp_escape_C
1710
- assert_scanned('/regex\\C-x/',
1711
- :tREGEXP_BEG, "/", [0, 1],
1712
- :tSTRING_CONTENT, "regex\\C-x", [1, 10],
1713
- :tSTRING_END, "/", [10, 11],
1714
- :tREGEXP_OPT, "", [11, 11])
1715
- end
1716
-
1717
- def test_regexp_escape_C_M
1718
- assert_scanned('/regex\\C-\\M-x/',
1719
- :tREGEXP_BEG, "/", [0, 1],
1720
- :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 13],
1721
- :tSTRING_END, "/", [13, 14],
1722
- :tREGEXP_OPT, "", [14, 14])
1723
- end
1724
-
1725
- def test_regexp_escape_C_M_craaaazy
1726
- assert_scanned("/regex\\C-\\\n\\M-x/",
1727
- :tREGEXP_BEG, "/", [0, 1],
1728
- :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 15],
1729
- :tSTRING_END, "/", [15, 16],
1730
- :tREGEXP_OPT, "", [16, 16])
1731
- end
1732
-
1733
- def test_regexp_escape_C_bad_dash
1734
- refute_scanned '/regex\\Cx/', :tREGEXP_BEG, "/", [0, 1]
1735
- end
1736
-
1737
- def test_regexp_escape_C_bad_dash_eos
1738
- refute_scanned '/regex\\C-/', :tREGEXP_BEG, "/", [0, 1]
1739
- end
1740
-
1741
- def test_regexp_escape_C_bad_dash_eos2
1742
- refute_scanned '/regex\\C-', :tREGEXP_BEG, "/", [0, 1]
1743
- end
1744
-
1745
- def test_regexp_escape_C_bad_eos
1746
- refute_scanned '/regex\\C/', :tREGEXP_BEG, "/", [0, 1]
1747
- end
1748
-
1749
- def test_regexp_escape_C_bad_eos2
1750
- refute_scanned '/regex\\c', :tREGEXP_BEG, "/", [0, 1]
1751
- end
1752
-
1753
- def test_regexp_escape_M
1754
- assert_scanned('/regex\\M-x/',
1755
- :tREGEXP_BEG, "/", [0, 1],
1756
- :tSTRING_CONTENT, "regex\\M-x", [1, 10],
1757
- :tSTRING_END, "/", [10, 11],
1758
- :tREGEXP_OPT, "", [11, 11])
1759
- end
1760
-
1761
- def test_regexp_escape_M_C
1762
- assert_scanned('/regex\\M-\\C-x/',
1763
- :tREGEXP_BEG, "/", [0, 1],
1764
- :tSTRING_CONTENT, "regex\\M-\\C-x", [1, 13],
1765
- :tSTRING_END, "/", [13, 14],
1766
- :tREGEXP_OPT, "", [14, 14])
1767
- end
1768
-
1769
- def test_regexp_escape_M_bad_dash
1770
- refute_scanned '/regex\\Mx/', :tREGEXP_BEG, "/", [0, 1]
1771
- end
1772
-
1773
- def test_regexp_escape_M_bad_dash_eos
1774
- refute_scanned '/regex\\M-/', :tREGEXP_BEG, "/", [0, 1]
1775
- end
1776
-
1777
- def test_regexp_escape_M_bad_dash_eos2
1778
- refute_scanned '/regex\\M-', :tREGEXP_BEG, "/", [0, 1]
1779
- end
1780
-
1781
- def test_regexp_escape_M_bad_eos
1782
- refute_scanned '/regex\\M/', :tREGEXP_BEG, "/", [0, 1]
1783
- end
1784
-
1785
- def test_regexp_escape_backslash_slash
1786
- assert_scanned('/\\//',
1787
- :tREGEXP_BEG, "/", [0, 1],
1788
- :tSTRING_CONTENT, '/', [1, 3],
1789
- :tSTRING_END, "/", [3, 4],
1790
- :tREGEXP_OPT, "", [4, 4])
1791
- end
1792
-
1793
- def test_regexp_escape_backslash_terminator
1794
- assert_scanned('%r%blah\\%blah%',
1795
- :tREGEXP_BEG, "%r%", [0, 3],
1796
- :tSTRING_CONTENT, "blah%blah", [3, 13],
1797
- :tSTRING_END, "%", [13, 14],
1798
- :tREGEXP_OPT, "", [14, 14])
1799
- end
1800
-
1801
- def test_regexp_escape_backslash_terminator_meta1
1802
- assert_scanned('%r{blah\\}blah}',
1803
- :tREGEXP_BEG, "%r{", [0, 3],
1804
- :tSTRING_CONTENT, "blah\\}blah", [3, 13],
1805
- :tSTRING_END, "}", [13, 14],
1806
- :tREGEXP_OPT, "", [14, 14])
1807
- end
1808
-
1809
- def test_regexp_escape_backslash_terminator_meta2
1810
- assert_scanned('%r/blah\\/blah/',
1811
- :tREGEXP_BEG, "%r/", [0, 3],
1812
- :tSTRING_CONTENT, "blah/blah", [3, 13],
1813
- :tSTRING_END, "/", [13, 14],
1814
- :tREGEXP_OPT, "", [14, 14])
1815
- end
1816
-
1817
- def test_regexp_escape_backslash_terminator_meta3
1818
- assert_scanned('%r/blah\\%blah/',
1819
- :tREGEXP_BEG, "%r/", [0, 3],
1820
- :tSTRING_CONTENT, "blah\\%blah", [3, 13],
1821
- :tSTRING_END, "/", [13, 14],
1822
- :tREGEXP_OPT, "", [14, 14])
1823
- end
1824
-
1825
- def test_regexp_escape_bad_eos
1826
- refute_scanned '/regex\\', :tREGEXP_BEG, "/", [0, 1]
1827
- end
1828
-
1829
- def test_regexp_escape_bs
1830
- assert_scanned('/regex\\\\regex/',
1831
- :tREGEXP_BEG, "/", [0, 1],
1832
- :tSTRING_CONTENT, "regex\\\\regex", [1, 13],
1833
- :tSTRING_END, "/", [13, 14],
1834
- :tREGEXP_OPT, "", [14, 14])
1835
- end
1836
-
1837
- def test_regexp_escape_c
1838
- assert_scanned('/regex\\cxxx/',
1839
- :tREGEXP_BEG, "/", [0, 1],
1840
- :tSTRING_CONTENT, "regex\\cxxx", [1, 11],
1841
- :tSTRING_END, "/", [11, 12],
1842
- :tREGEXP_OPT, "", [12, 12])
1843
- end
1844
-
1845
- def test_regexp_escape_c_backslash
1846
- assert_scanned('/regex\\c\\n/',
1847
- :tREGEXP_BEG, "/", [0, 1],
1848
- :tSTRING_CONTENT, "regex\\c\\n", [1, 10],
1849
- :tSTRING_END, "/", [10, 11],
1850
- :tREGEXP_OPT, "", [11, 11])
1851
- end
1852
-
1853
- def test_regexp_escape_chars
1854
- assert_scanned('/re\\tge\\nxp/',
1855
- :tREGEXP_BEG, "/", [0, 1],
1856
- :tSTRING_CONTENT, "re\\tge\\nxp", [1, 11],
1857
- :tSTRING_END, "/", [11, 12],
1858
- :tREGEXP_OPT, "", [12, 12])
1859
- end
1860
-
1861
- def test_regexp_escape_double_backslash
1862
- assert_scanned('/[\\/\\\\]$/',
1863
- :tREGEXP_BEG, "/", [0, 1],
1864
- :tSTRING_CONTENT,'[/\\\\]$', [1, 8],
1865
- :tSTRING_END, "/", [8, 9],
1866
- :tREGEXP_OPT, "", [9, 9])
1867
- end
1868
-
1869
- def test_regexp_escape_hex
1870
- assert_scanned('/regex\\x61xp/',
1871
- :tREGEXP_BEG, "/", [0, 1],
1872
- :tSTRING_CONTENT, "regex\\x61xp", [1, 12],
1873
- :tSTRING_END, "/", [12, 13],
1874
- :tREGEXP_OPT, "", [13, 13])
1875
- end
1876
-
1877
- def test_regexp_escape_hex_bad
1878
- refute_scanned '/regex\\xzxp/', :tREGEXP_BEG, "/", [0, 1]
1879
- end
1880
-
1881
- def test_regexp_escape_hex_one
1882
- assert_scanned('/^[\\xd\\xa]{2}/on',
1883
- :tREGEXP_BEG, '/', [0, 1],
1884
- :tSTRING_CONTENT, '^[\\xd\\xa]{2}', [1, 13],
1885
- :tSTRING_END, "/", [13, 14],
1886
- :tREGEXP_OPT, 'on', [14, 16])
1887
- end
1888
-
1889
- def test_regexp_escape_oct1
1890
- assert_scanned('/regex\\0xp/',
1891
- :tREGEXP_BEG, "/", [0, 1],
1892
- :tSTRING_CONTENT, "regex\\0xp", [1, 10],
1893
- :tSTRING_END, "/", [10, 11],
1894
- :tREGEXP_OPT, "", [11, 11])
1895
- end
1896
-
1897
- def test_regexp_escape_oct2
1898
- assert_scanned('/regex\\07xp/',
1899
- :tREGEXP_BEG, "/", [0, 1],
1900
- :tSTRING_CONTENT, "regex\\07xp", [1, 11],
1901
- :tSTRING_END, "/", [11, 12],
1902
- :tREGEXP_OPT, "", [12, 12])
1903
- end
1904
-
1905
- def test_regexp_escape_oct3
1906
- assert_scanned('/regex\\10142/',
1907
- :tREGEXP_BEG, "/", [0, 1],
1908
- :tSTRING_CONTENT, "regex\\10142", [1, 12],
1909
- :tSTRING_END, "/", [12, 13],
1910
- :tREGEXP_OPT, "", [13, 13])
1911
- end
1912
-
1913
- def test_regexp_escape_return
1914
- assert_scanned("/regex\\\nregex/",
1915
- :tREGEXP_BEG, "/", [0, 1],
1916
- :tSTRING_CONTENT, "regexregex", [1, 13],
1917
- :tSTRING_END, "/", [13, 14],
1918
- :tREGEXP_OPT, "", [14, 14])
1919
- end
1920
-
1921
- def test_regexp_escape_delimiter_meta
1922
- assert_scanned("%r(\\))",
1923
- :tREGEXP_BEG, "%r(", [0, 3],
1924
- :tSTRING_CONTENT, "\\)", [3, 5],
1925
- :tSTRING_END, ")", [5, 6],
1926
- :tREGEXP_OPT, "", [6, 6])
1927
- end
1928
-
1929
- def test_regexp_escape_delimiter_nonmeta
1930
- assert_scanned("%r'\\''",
1931
- :tREGEXP_BEG, "%r'", [0, 3],
1932
- :tSTRING_CONTENT, "'", [3, 5],
1933
- :tSTRING_END, "'", [5, 6],
1934
- :tREGEXP_OPT, "", [6, 6])
1935
- end
1936
-
1937
- def test_regexp_escape_other_meta
1938
- assert_scanned("/\\.\\$\\*\\+\\.\\?\\|/",
1939
- :tREGEXP_BEG, "/", [0, 1],
1940
- :tSTRING_CONTENT, "\\.\\$\\*\\+\\.\\?\\|", [1, 15],
1941
- :tSTRING_END, "/", [15, 16],
1942
- :tREGEXP_OPT, "", [16, 16])
1943
- end
1944
-
1945
- def test_regexp_nm
1946
- assert_scanned("/.*/nm",
1947
- :tREGEXP_BEG, "/", [0, 1],
1948
- :tSTRING_CONTENT, ".*", [1, 3],
1949
- :tSTRING_END, "/", [3, 4],
1950
- :tREGEXP_OPT, "nm", [4, 6])
1951
- end
1952
-
1953
- def test_rparen
1954
- assert_scanned ")", :tRPAREN, ")", [0, 1]
1955
- end
1956
-
1957
- def test_rshft
1958
- assert_scanned("a >> 2",
1959
- :tIDENTIFIER, "a", [0, 1],
1960
- :tRSHFT, ">>", [2, 4],
1961
- :tINTEGER, 2, [5, 6])
1962
- end
1963
-
1964
- def test_rshft_equals
1965
- assert_scanned("a >>= 2",
1966
- :tIDENTIFIER, "a", [0, 1],
1967
- :tOP_ASGN, ">>", [2, 5],
1968
- :tINTEGER, 2, [6, 7])
1969
- end
1970
-
1971
- def test_star
1972
- assert_scanned("a * ",
1973
- :tIDENTIFIER, "a", [0, 1],
1974
- :tSTAR2, "*", [2, 3])
1975
-
1976
- assert_equal :expr_value, @lex.state
1977
- end
1978
-
1979
- def test_star2
1980
- assert_scanned("a ** ",
1981
- :tIDENTIFIER, "a", [0, 1],
1982
- :tPOW, "**", [2, 4])
1983
-
1984
- assert_equal :expr_value, @lex.state
1985
- end
1986
-
1987
- def test_star2_equals
1988
- assert_scanned("a **= ",
1989
- :tIDENTIFIER, "a", [0, 1],
1990
- :tOP_ASGN, "**", [2, 5])
1991
-
1992
- assert_equal :expr_beg, @lex.state
1993
- end
1994
-
1995
- def test_star2_beg
1996
- assert_scanned("** ",
1997
- :tDSTAR, "**", [0, 2])
1998
-
1999
- assert_equal :expr_beg, @lex.state
2000
- end
2001
-
2002
- def test_star_arg
2003
- @lex.state = :expr_arg
2004
-
2005
- assert_scanned(" *a",
2006
- :tSTAR, "*", [1, 2],
2007
- :tIDENTIFIER, "a", [2, 3])
2008
-
2009
- assert_equal :expr_arg, @lex.state
2010
- end
2011
-
2012
- def test_star_arg_beg
2013
- @lex.state = :expr_beg
2014
-
2015
- assert_scanned("*a",
2016
- :tSTAR, "*", [0, 1],
2017
- :tIDENTIFIER, "a", [1, 2])
2018
-
2019
- assert_equal :expr_arg, @lex.state
2020
- end
2021
-
2022
- def test_star_arg_beg_fname
2023
- @lex.state = :expr_fname
2024
-
2025
- assert_scanned("*a",
2026
- :tSTAR2, "*", [0, 1],
2027
- :tIDENTIFIER, "a", [1, 2])
2028
-
2029
- assert_equal :expr_arg, @lex.state
2030
- end
2031
-
2032
- def test_star_equals
2033
- assert_scanned("a *= ",
2034
- :tIDENTIFIER, "a", [0, 1],
2035
- :tOP_ASGN, "*", [2, 4])
2036
-
2037
- assert_equal :expr_beg, @lex.state
2038
- end
2039
-
2040
- def test_string_bad_eos
2041
- refute_scanned('%',
2042
- :tSTRING_BEG, '%', [0, 1])
2043
- end
2044
-
2045
- def test_string_bad_eos_quote
2046
- refute_scanned('%{nest',
2047
- :tSTRING_BEG, '%}', [0, 2])
2048
- end
2049
-
2050
- def test_string_double
2051
- assert_scanned('"string"',
2052
- :tSTRING, "string", [0, 8])
2053
- end
2054
-
2055
- def test_string_double_escape_C
2056
- assert_scanned('"\\C-a"',
2057
- :tSTRING, "\001", [0, 6])
2058
- end
2059
-
2060
- def test_string_double_escape_C_backslash
2061
- assert_scanned('"\\C-\\\\"',
2062
- :tSTRING, "\034", [0, 7])
2063
- end
2064
-
2065
- def test_string_double_escape_C_escape
2066
- assert_scanned('"\\C-\\M-a"',
2067
- :tSTRING, "\201", [0, 9])
2068
- end
2069
-
2070
- def test_string_double_escape_C_question
2071
- assert_scanned('"\\C-?"',
2072
- :tSTRING, "\177", [0, 6])
2073
- end
2074
-
2075
- def test_string_double_escape_M
2076
- assert_scanned('"\\M-a"',
2077
- :tSTRING, "\341", [0, 6])
2078
- end
2079
-
2080
- def test_string_double_escape_M_backslash
2081
- assert_scanned('"\\M-\\\\"',
2082
- :tSTRING, "\334", [0, 7])
2083
- end
2084
-
2085
- def test_string_double_escape_M_escape
2086
- assert_scanned('"\\M-\\C-a"',
2087
- :tSTRING, "\201", [0, 9])
2088
- end
2089
-
2090
- def test_string_double_escape_bs1
2091
- assert_scanned('"a\\a\\a"',
2092
- :tSTRING, "a\a\a", [0, 7])
2093
- end
2094
-
2095
- def test_string_double_escape_bs2
2096
- assert_scanned('"a\\\\a"',
2097
- :tSTRING, "a\\a", [0, 6])
2098
- end
2099
-
2100
- def test_string_double_escape_c
2101
- assert_scanned('"\\ca"',
2102
- :tSTRING, "\001", [0, 5])
2103
- end
2104
-
2105
- def test_string_double_escape_c_escape
2106
- assert_scanned('"\\c\\M-a"',
2107
- :tSTRING, "\201", [0, 8])
2108
- end
2109
-
2110
- def test_string_double_escape_c_question
2111
- assert_scanned('"\\c?"',
2112
- :tSTRING, "\177", [0, 5])
2113
- end
2114
-
2115
- def test_string_double_escape_chars
2116
- assert_scanned('"s\\tri\\ng"',
2117
- :tSTRING, "s\tri\ng", [0, 10])
2118
- end
2119
-
2120
- def test_string_double_escape_hex
2121
- assert_scanned('"n = \\x61\\x62\\x63"',
2122
- :tSTRING, "n = abc", [0, 18])
2123
- end
2124
-
2125
- def test_string_double_escape_octal
2126
- assert_scanned('"n = \\101\\102\\103"',
2127
- :tSTRING, "n = ABC", [0, 18])
2128
- end
2129
-
2130
- def test_string_double_escape_octal_wrap
2131
- assert_scanned('"\\753"',
2132
- :tSTRING, "\xEB", [0, 6])
2133
- end
2134
-
2135
- def test_string_double_interp
2136
- assert_scanned("\"blah #x a \#@a b \#$b c \#{3} # \"",
2137
- :tSTRING_BEG, "\"", [0, 1],
2138
- :tSTRING_CONTENT, "blah #x a ", [1, 11],
2139
- :tSTRING_DVAR, nil, [11, 12],
2140
- :tIVAR, "@a", [12, 14],
2141
- :tSTRING_CONTENT, " b ", [14, 17],
2142
- :tSTRING_DVAR, nil, [17, 18],
2143
- :tGVAR, "$b", [18, 20],
2144
- :tSTRING_CONTENT, " c ", [20, 23],
2145
- :tSTRING_DBEG, '#{', [23, 25],
2146
- :tINTEGER, 3, [25, 26],
2147
- :tRCURLY, "}", [26, 27],
2148
- :tSTRING_CONTENT, " # ", [27, 30],
2149
- :tSTRING_END, "\"", [30, 31])
2150
- end
2151
-
2152
- def test_string_double_interp_label
2153
- assert_scanned('"#{foo:bar}"',
2154
- :tSTRING_BEG, '"', [0, 1],
2155
- :tSTRING_DBEG, '#{', [1, 3],
2156
- :tIDENTIFIER, 'foo', [3, 6],
2157
- :tSYMBOL, 'bar', [6, 10],
2158
- :tRCURLY, '}', [10, 11],
2159
- :tSTRING_END, '"', [11, 12])
2160
- end
2161
-
2162
- def test_string_double_nested_curlies
2163
- assert_scanned('%{nest{one{two}one}nest}',
2164
- :tSTRING_BEG, '%{', [0, 2],
2165
- :tSTRING_CONTENT, "nest{one{two}one}nest", [2, 23],
2166
- :tSTRING_END, '}', [23, 24])
2167
- end
2168
-
2169
- def test_string_double_no_interp
2170
- assert_scanned("\"# blah\"", # pound first
2171
- :tSTRING, "# blah", [0, 8])
2172
-
2173
- assert_scanned("\"blah # blah\"", # pound not first
2174
- :tSTRING, "blah # blah", [0, 13])
2175
- end
2176
-
2177
- def test_string_escape_x_single
2178
- assert_scanned('"\\x0"',
2179
- :tSTRING, "\000", [0, 5])
2180
- end
2181
-
2182
- def test_string_pct_Q
2183
- assert_scanned("%Q[s1 s2]",
2184
- :tSTRING_BEG, '%Q[', [0, 3],
2185
- :tSTRING_CONTENT, "s1 s2", [3, 8],
2186
- :tSTRING_END, ']', [8, 9])
2187
- end
2188
-
2189
- def test_string_pct_W
2190
- assert_scanned("%W[s1 s2\ns3]",
2191
- :tWORDS_BEG, "%W[", [0, 3],
2192
- :tSTRING_CONTENT, "s1", [3, 5],
2193
- :tSPACE, nil, [5, 6],
2194
- :tSTRING_CONTENT, "s2", [6, 8],
2195
- :tSPACE, nil, [8, 9],
2196
- :tSTRING_CONTENT, "s3", [9, 11],
2197
- :tSPACE, nil, [11, 11],
2198
- :tSTRING_END, ']', [11, 12])
2199
- end
2200
-
2201
- def test_string_pct_W_bs_nl
2202
- assert_scanned("%W[s1 \\\ns2]",
2203
- :tWORDS_BEG, "%W[", [0, 3],
2204
- :tSTRING_CONTENT, "s1", [3, 5],
2205
- :tSPACE, nil, [5, 6],
2206
- :tSTRING_CONTENT, "\ns2", [6, 10],
2207
- :tSPACE, nil, [10, 10],
2208
- :tSTRING_END, ']', [10, 11])
2209
- end
2210
-
2211
- def test_string_pct_W_interp
2212
- assert_scanned('%W[#{1}#{2} #@a]',
2213
- :tWORDS_BEG, '%W[', [0, 3],
2214
- :tSTRING_DBEG, '#{', [3, 5],
2215
- :tINTEGER, 1, [5, 6],
2216
- :tRCURLY, '}', [6, 7],
2217
- :tSTRING_DBEG, '#{', [7, 9],
2218
- :tINTEGER, 2, [9, 10],
2219
- :tRCURLY, '}', [10, 11],
2220
- :tSPACE, nil, [11, 12],
2221
- :tSTRING_DVAR, nil, [12, 13],
2222
- :tIVAR, '@a', [13, 15],
2223
- :tSPACE, nil, [15, 15],
2224
- :tSTRING_END, ']', [15, 16])
2225
- end
2226
-
2227
- def test_string_pct_I
2228
- assert_scanned("%I(s1 s2)",
2229
- :tSYMBOLS_BEG, "%I(", [0, 3],
2230
- :tSTRING_CONTENT, "s1", [3, 5],
2231
- :tSPACE, nil, [5, 6],
2232
- :tSTRING_CONTENT, "s2", [6, 8],
2233
- :tSPACE, nil, [8, 8],
2234
- :tSTRING_END, ')', [8, 9])
2235
- end
2236
-
2237
- def test_string_pct_angle
2238
- assert_scanned("%<blah>",
2239
- :tSTRING_BEG, '%<', [0, 2],
2240
- :tSTRING_CONTENT, "blah", [2, 6],
2241
- :tSTRING_END, '>', [6, 7])
2242
- end
2243
-
2244
- def test_string_pct_pct
2245
- assert_scanned("%%blah%",
2246
- :tSTRING_BEG, '%%', [0, 2],
2247
- :tSTRING_CONTENT, "blah", [2, 6],
2248
- :tSTRING_END, '%', [6, 7])
2249
- end
2250
-
2251
- def test_string_pct_w
2252
- assert_scanned("%w[s1 s2 ]",
2253
- :tQWORDS_BEG, "%w[", [0, 3],
2254
- :tSTRING_CONTENT, "s1", [3, 5],
2255
- :tSPACE, nil, [5, 6],
2256
- :tSTRING_CONTENT, "s2", [6, 8],
2257
- :tSPACE, nil, [8, 9],
2258
- :tSTRING_END, "]", [9, 10])
2259
- end
2260
-
2261
- def test_string_pct_w_incomplete
2262
- refute_scanned("%w[s1 ",
2263
- :tQWORDS_BEG, "%w[", [0, 3],
2264
- :tSTRING_CONTENT, "s1", [3, 5],
2265
- :tSPACE, nil, [5, 6])
2266
- end
2267
-
2268
- def test_string_pct_w_bs_nl
2269
- assert_scanned("%w[s1 \\\ns2]",
2270
- :tQWORDS_BEG, "%w[", [0, 3],
2271
- :tSTRING_CONTENT, "s1", [3, 5],
2272
- :tSPACE, nil, [5, 6],
2273
- :tSTRING_CONTENT, "\ns2", [6, 10],
2274
- :tSPACE, nil, [10, 10],
2275
- :tSTRING_END, ']', [10, 11])
2276
- end
2277
-
2278
- def test_string_pct_w_bs_sp
2279
- assert_scanned("%w[s\\ 1 s\\ 2]",
2280
- :tQWORDS_BEG, "%w[", [0, 3],
2281
- :tSTRING_CONTENT, "s 1", [3, 7],
2282
- :tSPACE, nil, [7, 8],
2283
- :tSTRING_CONTENT, "s 2", [8, 12],
2284
- :tSPACE, nil, [12, 12],
2285
- :tSTRING_END, ']', [12, 13])
2286
- end
2287
-
2288
- def test_string_pct_w_tab
2289
- assert_scanned("%w[abc\tdef]",
2290
- :tQWORDS_BEG, "%w[", [0, 3],
2291
- :tSTRING_CONTENT, "abc", [3, 6],
2292
- :tSPACE, nil, [6, 7],
2293
- :tSTRING_CONTENT, "def", [7, 10],
2294
- :tSPACE, nil, [10, 10],
2295
- :tSTRING_END, ']', [10, 11])
2296
- end
2297
-
2298
- def test_string_pct_i
2299
- assert_scanned("%i(s1 s2)",
2300
- :tQSYMBOLS_BEG, "%i(", [0, 3],
2301
- :tSTRING_CONTENT, "s1", [3, 5],
2302
- :tSPACE, nil, [5, 6],
2303
- :tSTRING_CONTENT, "s2", [6, 8],
2304
- :tSPACE, nil, [8, 8],
2305
- :tSTRING_END, ')', [8, 9])
2306
- end
2307
-
2308
- def test_string_pct_backslash
2309
- assert_scanned("%\\a\\",
2310
- :tSTRING_BEG, "%\\", [0, 2],
2311
- :tSTRING_CONTENT, "a", [2, 3],
2312
- :tSTRING_END, "\\", [3, 4])
2313
- end
2314
-
2315
- def test_string_pct_w_backslash
2316
- assert_scanned("%w\\s1 s2 \\",
2317
- :tQWORDS_BEG, "%w\\", [0, 3],
2318
- :tSTRING_CONTENT, "s1", [3, 5],
2319
- :tSPACE, nil, [5, 6],
2320
- :tSTRING_CONTENT, "s2", [6, 8],
2321
- :tSPACE, nil, [8, 9],
2322
- :tSTRING_END, "\\", [9, 10])
2323
- end
2324
-
2325
- def test_string_pct_w_backslash_nl
2326
- assert_scanned("%w\\s1 s2 \\\n",
2327
- :tQWORDS_BEG, "%w\\", [0, 3],
2328
- :tSTRING_CONTENT, "s1", [3, 5],
2329
- :tSPACE, nil, [5, 6],
2330
- :tSTRING_CONTENT, "s2", [6, 8],
2331
- :tSPACE, nil, [8, 9],
2332
- :tSTRING_END, "\\", [9, 10],
2333
- :tNL, nil, [10, 11])
2334
- end
2335
-
2336
- def test_string_pct_w_backslash_interp_nl
2337
- assert_scanned("%W\\blah #x a \#@a b \#$b c \#{3} # \\",
2338
- :tWORDS_BEG, "%W\\", [0, 3],
2339
- :tSTRING_CONTENT, "blah", [3, 7],
2340
- :tSPACE, nil, [7, 8],
2341
- :tSTRING_CONTENT, "#x", [8, 10],
2342
- :tSPACE, nil, [10, 11],
2343
- :tSTRING_CONTENT, "a", [11, 12],
2344
- :tSPACE, nil, [12, 13],
2345
- :tSTRING_DVAR, nil, [13, 14],
2346
- :tIVAR, "@a", [14, 16],
2347
- :tSPACE, nil, [16, 17],
2348
- :tSTRING_CONTENT, "b", [17, 18],
2349
- :tSPACE, nil, [18, 19],
2350
- :tSTRING_DVAR, nil, [19, 20],
2351
- :tGVAR, "$b", [20, 22],
2352
- :tSPACE, nil, [22, 23],
2353
- :tSTRING_CONTENT, "c", [23, 24],
2354
- :tSPACE, nil, [24, 25],
2355
- :tSTRING_DBEG, '#{', [25, 27],
2356
- :tINTEGER, 3, [27, 28],
2357
- :tRCURLY, "}", [28, 29],
2358
- :tSPACE, nil, [29, 30],
2359
- :tSTRING_CONTENT, "#", [30, 31],
2360
- :tSPACE, nil, [31, 32],
2361
- :tSTRING_END, "\\", [32, 33])
2362
- end
2363
-
2364
- def test_string_pct_backslash_with_bad_escape
2365
- # No escapes are allowed in a backslash-delimited string
2366
- refute_scanned("%\\a\\n\\",
2367
- :tSTRING_BEG, "%\\", [0, 2],
2368
- :tSTRING_CONTENT, "a", [2, 3],
2369
- :tSTRING_END, "\\", [3, 4],
2370
- :tIDENTIFIER, "n", [4, 5])
2371
- end
2372
-
2373
- def test_string_pct_intertwined_with_heredoc
2374
- assert_scanned("<<-foo + %\\a\nbar\nfoo\nb\\",
2375
- :tSTRING_BEG, "<<\"", [0, 6],
2376
- :tSTRING_CONTENT, "bar\n", [13, 17],
2377
- :tSTRING_END, "foo", [17, 20],
2378
- :tPLUS, "+", [7, 8],
2379
- :tSTRING_BEG, "%\\", [9, 11],
2380
- :tSTRING_CONTENT, "a\n", [11, 13],
2381
- :tSTRING_CONTENT, "b", [21, 22],
2382
- :tSTRING_END, "\\", [22, 23])
2383
- end
2384
-
2385
- def test_string_pct_q_backslash
2386
- assert_scanned("%q\\a\\",
2387
- :tSTRING_BEG, "%q\\", [0, 3],
2388
- :tSTRING_CONTENT, "a", [3, 4],
2389
- :tSTRING_END, "\\", [4, 5])
2390
- end
2391
-
2392
- def test_string_pct_Q_backslash
2393
- assert_scanned("%Q\\a\\",
2394
- :tSTRING_BEG, "%Q\\", [0, 3],
2395
- :tSTRING_CONTENT, "a", [3, 4],
2396
- :tSTRING_END, "\\", [4, 5])
2397
- end
2398
-
2399
- def test_string_single
2400
- assert_scanned("'string'",
2401
- :tSTRING, "string", [0, 8])
2402
- end
2403
-
2404
- def test_string_single_escape_chars
2405
- assert_scanned("'s\\tri\\ng'",
2406
- :tSTRING, "s\\tri\\ng", [0, 10])
2407
- end
2408
-
2409
- def test_string_single_nl
2410
- assert_scanned("'blah\\\nblah'",
2411
- :tSTRING_BEG, "'", [0, 1],
2412
- :tSTRING_CONTENT, "blah\\\n", [1, 7],
2413
- :tSTRING_CONTENT, "blah", [7, 11],
2414
- :tSTRING_END, "'", [11, 12])
2415
- end
2416
-
2417
- def test_symbol
2418
- assert_scanned(":symbol",
2419
- :tSYMBOL, "symbol", [0, 7])
2420
- end
2421
-
2422
- def test_symbol_double
2423
- assert_scanned(":\"symbol\"",
2424
- :tSYMBEG, ":\"", [0, 2],
2425
- :tSTRING_CONTENT, "symbol", [2, 8],
2426
- :tSTRING_END, "\"", [8, 9])
2427
- end
2428
-
2429
- def test_symbol_single
2430
- assert_scanned(":'symbol'",
2431
- :tSYMBEG, ":'", [0, 2],
2432
- :tSTRING_CONTENT, "symbol", [2, 8],
2433
- :tSTRING_END, "'", [8, 9])
2434
- end
2435
-
2436
- def test_ternary
2437
- assert_scanned("a ? b : c",
2438
- :tIDENTIFIER, "a", [0, 1],
2439
- :tEH, "?", [2, 3],
2440
- :tIDENTIFIER, "b", [4, 5],
2441
- :tCOLON, ":", [6, 7],
2442
- :tIDENTIFIER, "c", [8, 9])
2443
-
2444
- assert_scanned("a ?b : c",
2445
- :tIDENTIFIER, "a", [0, 1],
2446
- :tINTEGER, 98, [2, 4],
2447
- :tCOLON, ":", [5, 6],
2448
- :tIDENTIFIER, "c", [7, 8])
2449
-
2450
- assert_scanned("a ?bb : c", # GAH! MATZ!!!
2451
- :tIDENTIFIER, "a", [0, 1],
2452
- :tEH, "?", [2, 3],
2453
- :tIDENTIFIER, "bb", [3, 5],
2454
- :tCOLON, ":", [6, 7],
2455
- :tIDENTIFIER, "c", [8, 9])
2456
-
2457
- assert_scanned("42 ?", # 42 forces expr_end
2458
- :tINTEGER, 42, [0, 2],
2459
- :tEH, "?", [3, 4])
2460
- end
2461
-
2462
- def test_tilde
2463
- assert_scanned "~", :tTILDE, "~", [0, 1]
2464
- end
2465
-
2466
- def test_tilde_unary
2467
- @lex.state = :expr_fname
2468
- assert_scanned "~@", :tTILDE, "~@", [0, 2]
2469
- end
2470
-
2471
- def test_uminus
2472
- assert_scanned("-blah",
2473
- :tUMINUS, "-", [0, 1],
2474
- :tIDENTIFIER, "blah", [1, 5])
2475
- end
2476
-
2477
- def test_underscore
2478
- assert_scanned("_var", :tIDENTIFIER, "_var", [0, 4])
2479
- end
2480
-
2481
- def test_underscore_end
2482
- assert_scanned("__END__\n")
2483
- assert_scanned("__END__")
2484
- assert_scanned("__END__ foo",
2485
- :tIDENTIFIER, '__END__', [0, 7],
2486
- :tIDENTIFIER, 'foo', [8, 11])
2487
- assert_scanned("__END__\rfoo",
2488
- :tIDENTIFIER, '__END__', [0, 7],
2489
- :tIDENTIFIER, 'foo', [8, 11])
2490
- end
2491
-
2492
- def test_uplus
2493
- assert_scanned("+blah",
2494
- :tUPLUS, "+", [0, 1],
2495
- :tIDENTIFIER, "blah", [1, 5])
2496
- end
2497
-
2498
- def test_if_unless_mod
2499
- assert_scanned("return if true unless false",
2500
- :kRETURN, "return", [0, 6],
2501
- :kIF_MOD, "if", [7, 9],
2502
- :kTRUE, "true", [10, 14],
2503
- :kUNLESS_MOD, "unless", [15, 21],
2504
- :kFALSE, "false", [22, 27])
2505
- end
2506
-
2507
- def test_if_stmt
2508
- assert_scanned("if true\n return end",
2509
- :kIF, "if", [0, 2],
2510
- :kTRUE, "true", [3, 7],
2511
- :tNL, nil, [7, 8],
2512
- :kRETURN, "return", [9, 15],
2513
- :kEND, "end", [16, 19])
2514
- end
2515
-
2516
- def test_sclass_label
2517
- setup_lexer 20
2518
- assert_scanned("class << a:b",
2519
- :kCLASS, 'class', [0, 5],
2520
- :tLSHFT, '<<', [6, 8],
2521
- :tIDENTIFIER, 'a', [9, 10],
2522
- :tSYMBOL, 'b', [10, 12])
2523
- end
2524
-
2525
- def test_fname_pct_s__22
2526
- setup_lexer 22
2527
- @lex.state = :expr_fname
2528
- assert_scanned("%s(a)",
2529
- :tPERCENT, '%', [0, 1],
2530
- :tIDENTIFIER, 's', [1, 2],
2531
- :tLPAREN2, '(', [2, 3],
2532
- :tIDENTIFIER, 'a', [3, 4],
2533
- :tRPAREN, ')', [4, 5])
2534
- end
2535
-
2536
- def test_fname_pct_s__23
2537
- setup_lexer 23
2538
- @lex.state = :expr_fname
2539
- assert_scanned("%s(a)",
2540
- :tSYMBEG, '%s(', [0, 3],
2541
- :tSTRING_CONTENT, 'a', [3, 4],
2542
- :tSTRING_END, ')', [4, 5])
2543
- end
2544
-
2545
- def test_static_env
2546
- env = Parser::StaticEnvironment.new
2547
- env.declare "a"
2548
-
2549
- @lex.static_env = env
2550
- assert_scanned("a [42]",
2551
- :tIDENTIFIER, "a", [0, 1],
2552
- :tLBRACK2, "[", [2, 3],
2553
- :tINTEGER, 42, [3, 5],
2554
- :tRBRACK, "]", [5, 6])
2555
- end
2556
-
2557
- def test_int_suffix
2558
- [18, 19, 20].each do |version|
2559
- setup_lexer version
2560
-
2561
- assert_scanned("42r",
2562
- :tINTEGER, 42, [0, 2],
2563
- :tIDENTIFIER, 'r', [2, 3])
2564
-
2565
- assert_scanned("42if",
2566
- :tINTEGER, 42, [0, 2],
2567
- :kIF_MOD, 'if', [2, 4])
2568
- end
2569
-
2570
- setup_lexer 21
2571
-
2572
- assert_scanned("42r", :tRATIONAL, Rational(42), [0, 3])
2573
- assert_scanned("42i", :tIMAGINARY, Complex(0, 42), [0, 3])
2574
- assert_scanned("42ri", :tIMAGINARY, Complex(0, Rational(42)), [0, 4])
2575
- end
2576
-
2577
- def test_float_suffix
2578
- [18, 19, 20].each do |version|
2579
- setup_lexer version
2580
-
2581
- assert_scanned("42.1r",
2582
- :tFLOAT, 42.1, [0, 4],
2583
- :tIDENTIFIER, 'r', [4, 5])
2584
-
2585
- assert_scanned("42.1if",
2586
- :tFLOAT, 42.1, [0, 4],
2587
- :kIF_MOD, 'if', [4, 6])
2588
-
2589
- assert_scanned("1e1r",
2590
- :tFLOAT, 1e1, [0, 3],
2591
- :tIDENTIFIER, 'r', [3, 4])
2592
- end
2593
-
2594
- begin
2595
- # Feature-check.
2596
- Rational("10")
2597
-
2598
- setup_lexer 21
2599
-
2600
- assert_scanned("42.1r", :tRATIONAL, Rational(421, 10), [0, 5])
2601
- assert_scanned("42.1i", :tIMAGINARY, Complex(0, 42.1), [0, 5])
2602
- assert_scanned("42.1ri", :tIMAGINARY, Complex(0, Rational(421, 10)), [0, 6])
2603
- assert_scanned("42.1ir",
2604
- :tIMAGINARY, Complex(0, 42.1), [0, 5],
2605
- :tIDENTIFIER, 'r', [5, 6])
2606
-
2607
- assert_scanned("1e1i", :tIMAGINARY, Complex(0, 1e1), [0, 4])
2608
- assert_scanned("1e1r",
2609
- :tFLOAT, 1e1, [0, 3],
2610
- :tIDENTIFIER, 'r', [3, 4])
2611
- assert_scanned("1e1ri",
2612
- :tFLOAT, 1e1, [0, 3],
2613
- :tIDENTIFIER, 'ri', [3, 5])
2614
- assert_scanned("1e1ir",
2615
- :tIMAGINARY, Complex(0, 1e1), [0, 4],
2616
- :tIDENTIFIER, 'r', [4, 5])
2617
- rescue NoMethodError
2618
- # Ruby not modern enough
2619
- end
2620
- end
2621
-
2622
- def test_eof
2623
- assert_scanned("self",
2624
- :kSELF, "self", [0, 4])
2625
- assert_equal([false, ["$eof", Parser::Source::Range.new(@lex.source_buffer, 4, 4)]],
2626
- @lex.advance)
2627
- end
2628
-
2629
- #
2630
- # Test for 'fluent interface'
2631
- #
2632
-
2633
- def test_fluent_dot
2634
- assert_scanned("x\n.y",
2635
- :tIDENTIFIER, 'x', [0, 1],
2636
- :tDOT, '.', [2, 3],
2637
- :tIDENTIFIER, 'y', [3, 4])
2638
-
2639
- assert_scanned("x\n .y",
2640
- :tIDENTIFIER, 'x', [0, 1],
2641
- :tDOT, '.', [4, 5],
2642
- :tIDENTIFIER, 'y', [5, 6])
2643
-
2644
- assert_scanned("x # comment\n .y",
2645
- :tIDENTIFIER, 'x', [0, 1],
2646
- :tDOT, '.', [14, 15],
2647
- :tIDENTIFIER, 'y', [15, 16])
2648
- end
2649
-
2650
- def test_fluent_and_dot
2651
- assert_scanned("x\n&.y",
2652
- :tIDENTIFIER, 'x', [0, 1],
2653
- :tANDDOT, '&.', [2, 4],
2654
- :tIDENTIFIER, 'y', [4, 5])
2655
- end
2656
-
2657
- #
2658
- # Tests for whitespace.
2659
- #
2660
-
2661
- def test_whitespace_fname
2662
- @lex.state = :expr_fname
2663
- assert_scanned('class',
2664
- :kCLASS, 'class', [0, 5])
2665
-
2666
- @lex.state = :expr_fname
2667
- assert_scanned(' class',
2668
- :kCLASS, 'class', [1, 6])
2669
-
2670
- @lex.state = :expr_fname
2671
- assert_scanned("\nclass",
2672
- :kCLASS, 'class', [1, 6])
2673
-
2674
- @lex.state = :expr_fname
2675
- assert_scanned("\\\nclass",
2676
- :kCLASS, 'class', [2, 7])
2677
-
2678
- @lex.state = :expr_fname
2679
- assert_scanned("#foo\nclass",
2680
- :kCLASS, 'class', [5, 10])
2681
- end
2682
-
2683
- def test_whitespace_endfn
2684
- setup_lexer(21)
2685
-
2686
- @lex.state = :expr_endfn
2687
- assert_scanned('foo:',
2688
- :tLABEL, 'foo', [0, 4])
2689
-
2690
- @lex.state = :expr_endfn
2691
- assert_scanned(' foo:',
2692
- :tLABEL, 'foo', [1, 5])
2693
-
2694
- @lex.state = :expr_endfn
2695
- assert_scanned("\nfoo:",
2696
- :tNL, nil, [0, 1],
2697
- :tIDENTIFIER, 'foo', [1, 4],
2698
- :tCOLON, ':', [4, 5])
2699
-
2700
- @lex.state = :expr_endfn
2701
- assert_scanned("\nfoo: ",
2702
- :tNL, nil, [0, 1],
2703
- :tIDENTIFIER, 'foo', [1, 4],
2704
- :tCOLON, ':', [4, 5])
2705
-
2706
- @lex.state = :expr_endfn
2707
- assert_scanned("\\\nfoo:",
2708
- :tLABEL, 'foo', [2, 6])
2709
-
2710
- @lex.state = :expr_endfn
2711
- assert_scanned("#foo\nfoo:",
2712
- :tNL, nil, [4, 5],
2713
- :tIDENTIFIER, 'foo', [5, 8],
2714
- :tCOLON, ':', [8, 9])
2715
-
2716
- @lex.state = :expr_endfn
2717
- assert_scanned("#foo\nfoo: ",
2718
- :tNL, nil, [4, 5],
2719
- :tIDENTIFIER, 'foo', [5, 8],
2720
- :tCOLON, ':', [8, 9])
2721
- end
2722
-
2723
- def test_whitespace_dot
2724
- @lex.state = :expr_dot
2725
- assert_scanned('class',
2726
- :tIDENTIFIER, 'class', [0, 5])
2727
-
2728
- @lex.state = :expr_dot
2729
- assert_scanned(' class',
2730
- :tIDENTIFIER, 'class', [1, 6])
2731
-
2732
- @lex.state = :expr_dot
2733
- assert_scanned("\nclass",
2734
- :tIDENTIFIER, 'class', [1, 6])
2735
-
2736
- @lex.state = :expr_dot
2737
- assert_scanned("\\\nclass",
2738
- :tIDENTIFIER, 'class', [2, 7])
2739
-
2740
- @lex.state = :expr_dot
2741
- assert_scanned("#foo\nclass",
2742
- :tIDENTIFIER, 'class', [5, 10])
2743
- end
2744
-
2745
- def test_whitespace_arg
2746
- @lex.state = :expr_arg
2747
- assert_scanned('+',
2748
- :tPLUS, '+', [0, 1])
2749
-
2750
- @lex.state = :expr_arg
2751
- assert_scanned(' +',
2752
- :tUPLUS, '+', [1, 2])
2753
-
2754
- @lex.state = :expr_arg
2755
- assert_scanned("\n+",
2756
- :tNL, nil, [0, 1],
2757
- :tUPLUS, '+', [1, 2])
2758
-
2759
- @lex.state = :expr_arg
2760
- assert_scanned("\\\n+",
2761
- :tUPLUS, '+', [2, 3])
2762
-
2763
- @lex.state = :expr_arg
2764
- assert_scanned("\\\n +",
2765
- :tUPLUS, '+', [3, 4])
2766
-
2767
- @lex.state = :expr_arg
2768
- assert_scanned("#foo\n+",
2769
- :tNL, nil, [4, 5],
2770
- :tUPLUS, '+', [5, 6])
2771
- end
2772
-
2773
- def test_whitespace_endarg
2774
- @lex.state = :expr_endarg
2775
- assert_scanned('{',
2776
- :tLBRACE_ARG, '{', [0, 1])
2777
-
2778
- @lex.state = :expr_endarg
2779
- assert_scanned(' {',
2780
- :tLBRACE_ARG, '{', [1, 2])
2781
-
2782
- @lex.state = :expr_endarg
2783
- assert_scanned("\n{",
2784
- :tNL, nil, [0, 1],
2785
- :tLBRACE, '{', [1, 2])
2786
-
2787
- @lex.state = :expr_endarg
2788
- assert_scanned("\\\n{",
2789
- :tLBRACE_ARG, '{', [2, 3])
2790
-
2791
- @lex.state = :expr_endarg
2792
- assert_scanned("#foo\n{",
2793
- :tNL, nil, [4, 5],
2794
- :tLBRACE, '{', [5, 6])
2795
- end
2796
-
2797
- def test_whitespace_mid
2798
- @lex.state = :expr_mid
2799
- assert_scanned('+',
2800
- :tUPLUS, '+', [0, 1])
2801
-
2802
- @lex.state = :expr_mid
2803
- assert_scanned(' +',
2804
- :tUPLUS, '+', [1, 2])
2805
-
2806
- @lex.state = :expr_mid
2807
- assert_scanned("\n+",
2808
- :tNL, nil, [0, 1],
2809
- :tUPLUS, '+', [1, 2])
2810
-
2811
- @lex.state = :expr_mid
2812
- assert_scanned("\\\n+",
2813
- :tUPLUS, '+', [2, 3])
2814
-
2815
- @lex.state = :expr_mid
2816
- assert_scanned("#foo\n+",
2817
- :tNL, nil, [4, 5],
2818
- :tUPLUS, '+', [5, 6])
2819
- end
2820
-
2821
- def test_whitespace_beg
2822
- @lex.state = :expr_beg
2823
- assert_scanned('+',
2824
- :tUPLUS, '+', [0, 1])
2825
-
2826
- @lex.state = :expr_beg
2827
- assert_scanned(' +',
2828
- :tUPLUS, '+', [1, 2])
2829
-
2830
- @lex.state = :expr_beg
2831
- assert_scanned("\n+",
2832
- :tUPLUS, '+', [1, 2])
2833
-
2834
- @lex.state = :expr_beg
2835
- assert_scanned("\\\n+",
2836
- :tUPLUS, '+', [2, 3])
2837
-
2838
- @lex.state = :expr_beg
2839
- assert_scanned("#foo\n+",
2840
- :tUPLUS, '+', [5, 6])
2841
- end
2842
-
2843
- def test_whitespace_value
2844
- setup_lexer(20)
2845
-
2846
- @lex.state = :expr_value
2847
- assert_scanned('a:b',
2848
- :tIDENTIFIER, 'a', [0, 1],
2849
- :tSYMBOL, 'b', [1, 3])
2850
-
2851
- @lex.state = :expr_value
2852
- assert_scanned(' a:b',
2853
- :tIDENTIFIER, 'a', [1, 2],
2854
- :tSYMBOL, 'b', [2, 4])
2855
-
2856
- @lex.state = :expr_value
2857
- assert_scanned("\na:b",
2858
- :tIDENTIFIER, 'a', [1, 2],
2859
- :tSYMBOL, 'b', [2, 4])
2860
-
2861
- @lex.state = :expr_value
2862
- assert_scanned("\\\na:b",
2863
- :tIDENTIFIER, 'a', [2, 3],
2864
- :tSYMBOL, 'b', [3, 5])
2865
-
2866
- @lex.state = :expr_value
2867
- assert_scanned("#foo\na:b",
2868
- :tIDENTIFIER, 'a', [5, 6],
2869
- :tSYMBOL, 'b', [6, 8])
2870
- end
2871
-
2872
- def test_whitespace_end
2873
- @lex.state = :expr_end
2874
- assert_scanned('+ 1',
2875
- :tPLUS, '+', [0, 1],
2876
- :tINTEGER, 1, [2, 3])
2877
-
2878
- @lex.state = :expr_end
2879
- assert_scanned(' + 1',
2880
- :tPLUS, '+', [1, 2],
2881
- :tINTEGER, 1, [3, 4])
2882
-
2883
- @lex.state = :expr_end
2884
- assert_scanned("\n+ 1",
2885
- :tNL, nil, [0, 1],
2886
- :tUNARY_NUM, '+', [1, 2],
2887
- :tINTEGER, 1, [3, 4])
2888
-
2889
- @lex.state = :expr_end
2890
- assert_scanned("\\\n+ 1",
2891
- :tPLUS, '+', [2, 3],
2892
- :tINTEGER, 1, [4, 5])
2893
-
2894
- @lex.state = :expr_end
2895
- assert_scanned("#foo\n+ 1",
2896
- :tNL, nil, [4, 5],
2897
- :tUNARY_NUM, '+', [5, 6],
2898
- :tINTEGER, 1, [7, 8])
2899
- end
2900
-
2901
- def test_whitespace_cr
2902
- setup_lexer(20)
2903
- assert_scanned("<<E\nfoo\nE\rO",
2904
- :tSTRING_BEG, '<<"', [0, 3],
2905
- :tSTRING_CONTENT, "foo\n", [4, 8],
2906
- :tSTRING_END, 'E', [8, 11],
2907
- :tNL, nil, [3, 4])
2908
-
2909
- setup_lexer(21)
2910
- refute_scanned("<<E\nfoo\nE\rO",
2911
- :tSTRING_BEG, '<<"', [0, 3],
2912
- :tSTRING_CONTENT, "foo\n", [4, 8])
2913
- end
2914
-
2915
- #
2916
- # Handling of encoding-related issues.
2917
- #
2918
-
2919
- def test_transcoded_source_is_converted_back_to_original_encoding
2920
- setup_lexer(19)
2921
- @lex.force_utf32 = true
2922
- @lex.tokens = []
2923
- assert_scanned(utf('"a" + "b"'),
2924
- :tSTRING, "a", [0, 3],
2925
- :tPLUS, "+", [4, 5],
2926
- :tSTRING, "b", [6, 9])
2927
-
2928
- @lex.tokens.each do |_type, (str, _range)|
2929
- assert_equal Encoding::UTF_8, str.encoding
2930
- end
2931
- end
2932
-
2933
- #
2934
- # Tests for bugs.
2935
- #
2936
- # These tests should be moved from nursery and properly
2937
- # categorized when it's clear how to do that.
2938
- #
2939
-
2940
- def test_bug_sclass_joined
2941
- assert_scanned("class<<self",
2942
- :kCLASS, "class", [0, 5],
2943
- :tLSHFT, "<<", [5, 7],
2944
- :kSELF, "self", [7, 11])
2945
- end
2946
-
2947
- def test_bug_const_expr_end
2948
- assert_scanned("Option",
2949
- :tCONSTANT, 'Option', [0, 6])
2950
-
2951
- assert_equal :expr_cmdarg, @lex.state
2952
- end
2953
-
2954
- def test_bug_expr_beg_div
2955
- @lex.state = :expr_beg
2956
- assert_scanned("/=/",
2957
- :tREGEXP_BEG, "/", [0, 1],
2958
- :tSTRING_CONTENT, "=", [1, 2],
2959
- :tSTRING_END, "/", [2, 3],
2960
- :tREGEXP_OPT, "", [3, 3])
2961
-
2962
- @lex.state = :expr_beg
2963
- assert_scanned("/ = /",
2964
- :tREGEXP_BEG, "/", [0, 1],
2965
- :tSTRING_CONTENT, " = ", [1, 4],
2966
- :tSTRING_END, "/", [4, 5],
2967
- :tREGEXP_OPT, "", [5, 5])
2968
- end
2969
-
2970
- def test_bug_expr_beg_percent
2971
- @lex.state = :expr_beg
2972
- assert_scanned("%=foo=",
2973
- :tSTRING_BEG, "%=", [0, 2],
2974
- :tSTRING_CONTENT, 'foo', [2, 5],
2975
- :tSTRING_END, "=", [5, 6])
2976
-
2977
- @lex.state = :expr_beg
2978
- assert_scanned("% = ",
2979
- :tSTRING_BEG, "% ", [0, 2],
2980
- :tSTRING_CONTENT, '=', [2, 3],
2981
- :tSTRING_END, ' ', [3, 4])
2982
- end
2983
-
2984
- def test_bug_expr_beg_document
2985
- @lex.state = :expr_beg
2986
- assert_scanned(" \n=begin\n=end\nend",
2987
- :kEND, "end", [14, 17])
2988
-
2989
- end
2990
-
2991
- def test_bug_expr_beg_number
2992
- @lex.state = :expr_beg
2993
- assert_scanned("86400_000_000",
2994
- :tINTEGER, 86400_000_000, [0, 13])
2995
- end
2996
-
2997
- def test_bug_expr_beg_backspace_nl
2998
- @lex.state = :expr_beg
2999
- assert_scanned("\n/foo/",
3000
- :tREGEXP_BEG, "/", [1, 2],
3001
- :tSTRING_CONTENT, "foo", [2, 5],
3002
- :tSTRING_END, "/", [5, 6],
3003
- :tREGEXP_OPT, "", [6, 6])
3004
- end
3005
-
3006
- def test_bug_expr_beg_heredoc
3007
- assert_scanned("<<EOL % [\nfoo\nEOL\n]",
3008
- :tSTRING_BEG, '<<"', [0, 5],
3009
- :tSTRING_CONTENT, "foo\n", [10, 14],
3010
- :tSTRING_END, 'EOL', [14, 17],
3011
- :tPERCENT, '%', [6, 7],
3012
- :tLBRACK, '[', [8, 9],
3013
- :tRBRACK, ']', [18, 19])
3014
- end
3015
-
3016
- def test_bug_expr_beg_fid
3017
- assert_scanned("Rainbows!",
3018
- :tFID, 'Rainbows!', [0, 9])
3019
- end
3020
-
3021
- def test_bug_expr_beg_rescue_assoc
3022
- assert_scanned("rescue=>",
3023
- :kRESCUE, 'rescue', [0, 6],
3024
- :tASSOC, '=>', [6, 8])
3025
- end
3026
-
3027
- def test_bug_expr_arg_percent
3028
- @lex.state = :expr_arg
3029
- assert_scanned("%[",
3030
- :tPERCENT, "%", [0, 1],
3031
- :tLBRACK, "[", [1, 2])
3032
-
3033
- @lex.state = :expr_arg
3034
- assert_scanned("%=1",
3035
- :tOP_ASGN, "%", [0, 2],
3036
- :tINTEGER, 1, [2, 3])
3037
-
3038
- @lex.state = :expr_arg
3039
- assert_scanned(" %[1]",
3040
- :tSTRING_BEG, "%[", [1, 3],
3041
- :tSTRING_CONTENT, '1', [3, 4],
3042
- :tSTRING_END, ']', [4, 5])
3043
-
3044
- @lex.state = :expr_arg
3045
- assert_scanned(" %=1=",
3046
- :tOP_ASGN, "%", [1, 3],
3047
- :tINTEGER, 1, [3, 4],
3048
- :tEQL, "=", [4, 5])
3049
-
3050
- @lex.state = :expr_arg
3051
- assert_scanned(" %\n",
3052
- :tPERCENT, '%', [1, 2])
3053
- end
3054
-
3055
- def test_bug_expr_arg_lt_lt
3056
- @lex.state = :expr_arg
3057
- assert_scanned("<<EOS\nEOS",
3058
- :tLSHFT, "<<", [0, 2],
3059
- :tCONSTANT, "EOS", [2, 5],
3060
- :tNL, nil, [5, 6],
3061
- :tCONSTANT, "EOS", [6, 9])
3062
-
3063
- @lex.state = :expr_arg
3064
- assert_scanned(" <<EOS\nEOS",
3065
- :tSTRING_BEG, "<<\"", [1, 6],
3066
- :tSTRING_END, "EOS", [7, 10],
3067
- :tNL, nil, [6, 7])
3068
- end
3069
-
3070
- def test_bug_expr_arg_slash
3071
- @lex.state = :expr_arg
3072
- assert_scanned("/1",
3073
- :tDIVIDE, "/", [0, 1],
3074
- :tINTEGER, 1, [1, 2])
3075
-
3076
- @lex.state = :expr_arg
3077
- assert_scanned("/ 1",
3078
- :tDIVIDE, "/", [0, 1],
3079
- :tINTEGER, 1, [2, 3])
3080
-
3081
- @lex.state = :expr_arg
3082
- assert_scanned(" /1/",
3083
- :tREGEXP_BEG, "/", [1, 2],
3084
- :tSTRING_CONTENT, "1", [2, 3],
3085
- :tSTRING_END, "/", [3, 4],
3086
- :tREGEXP_OPT, "", [4, 4])
3087
-
3088
- @lex.state = :expr_arg
3089
- assert_scanned(" / 1",
3090
- :tDIVIDE, "/", [1, 2],
3091
- :tINTEGER, 1, [3, 4])
3092
- end
3093
-
3094
- def test_bug_expr_arg_label
3095
- setup_lexer 19
3096
-
3097
- @lex.state = :expr_arg
3098
- assert_scanned(" unless:",
3099
- :tLABEL, 'unless', [1, 8])
3100
-
3101
- @lex.state = :expr_arg
3102
- assert_scanned(" unless: ",
3103
- :tLABEL, 'unless', [1, 8])
3104
- end
3105
-
3106
- def test_bug_heredoc_continuation
3107
- @lex.state = :expr_arg
3108
- assert_scanned(" <<EOS\nEOS\nend",
3109
- :tSTRING_BEG, "<<\"", [1, 6],
3110
- :tSTRING_END, "EOS", [7, 10],
3111
- :tNL, nil, [6, 7],
3112
- :kEND, "end", [11, 14])
3113
- end
3114
-
3115
- def test_bug_heredoc_cr_lf
3116
- assert_scanned("<<FIN\r\nfoo\r\nFIN\r\n",
3117
- :tSTRING_BEG, "<<\"", [0, 5],
3118
- :tSTRING_CONTENT, "foo\n", [6, 10],
3119
- :tSTRING_END, "FIN", [10, 13],
3120
- :tNL, nil, [5, 6])
3121
- end
3122
-
3123
- def test_bug_eh_symbol_no_newline
3124
- assert_scanned("?\"\nfoo",
3125
- :tINTEGER, 34, [0, 2],
3126
- :tNL, nil, [2, 3],
3127
- :tIDENTIFIER, "foo", [3, 6])
3128
- end
3129
-
3130
- def test_bug_expr_arg_newline
3131
- @lex.state = :expr_arg
3132
- assert_scanned("\nfoo",
3133
- :tNL, nil, [0, 1],
3134
- :tIDENTIFIER, "foo", [1, 4])
3135
-
3136
- @lex.state = :expr_arg
3137
- assert_scanned(" \nfoo",
3138
- :tNL, nil, [1, 2],
3139
- :tIDENTIFIER, "foo", [2, 5])
3140
-
3141
- @lex.state = :expr_arg
3142
- assert_scanned("#foo\nfoo",
3143
- :tNL, nil, [4, 5],
3144
- :tIDENTIFIER, "foo", [5, 8])
3145
- end
3146
-
3147
- def test_bug_expr_arg_comment_newline
3148
- @lex.state = :expr_arg
3149
- assert_scanned(" #\nfoo",
3150
- :tNL, nil, [2, 3],
3151
- :tIDENTIFIER, 'foo', [3, 6])
3152
- end
3153
-
3154
- def test_bug_expr_arg_eh_crlf
3155
- @lex.state = :expr_arg
3156
- assert_scanned(" ?\r\n",
3157
- :tEH, '?', [1, 2])
3158
- end
3159
-
3160
- def test_bug_heredoc_backspace_nl
3161
- assert_scanned(" <<'XXX'\nf \\\nXXX\n",
3162
- :tSTRING_BEG, "<<'", [1, 8],
3163
- :tSTRING_CONTENT, "f \\\n", [9, 13],
3164
- :tSTRING_END, "XXX", [13, 16],
3165
- :tNL, nil, [8, 9])
3166
- end
3167
-
3168
- def test_bug_heredoc_lshft
3169
- assert_scanned("<<RULES << CLEANINGS\nRULES",
3170
- :tSTRING_BEG, '<<"', [0, 7],
3171
- :tSTRING_END, 'RULES', [21, 26],
3172
- :tLSHFT, '<<', [8, 10],
3173
- :tCONSTANT, 'CLEANINGS', [11, 20])
3174
- end
3175
-
3176
- def test_bug_sclass_comment_lshft_label
3177
- assert_scanned("class # foo\n<< a:b;end",
3178
- :kCLASS, 'class', [0, 5],
3179
- :tLSHFT, '<<', [12, 14],
3180
- :tIDENTIFIER, 'a', [15, 16],
3181
- :tSYMBOL, 'b', [16, 18],
3182
- :tSEMI, ';', [18, 19],
3183
- :kEND, 'end', [19, 22])
3184
- end
3185
-
3186
- def test_bug_expr_dot_comment
3187
- assert_scanned("foo. #bar\nbaz",
3188
- :tIDENTIFIER, 'foo', [0, 3],
3189
- :tDOT, '.', [3, 4],
3190
- :tIDENTIFIER, 'baz', [10, 13])
3191
- end
3192
-
3193
- def test_bug_expr_dot_fid
3194
- assert_scanned("foo.S?",
3195
- :tIDENTIFIER, 'foo', [0, 3],
3196
- :tDOT, '.', [3, 4],
3197
- :tFID, 'S?', [4, 6])
3198
- end
3199
-
3200
- def test_bug_expr_dot_id_eq
3201
- assert_scanned("foo.x= 1",
3202
- :tIDENTIFIER, 'foo', [0, 3],
3203
- :tDOT, '.', [3, 4],
3204
- :tIDENTIFIER, 'x', [4, 5],
3205
- :tEQL, '=', [5, 6],
3206
- :tINTEGER, 1, [7, 8])
3207
- end
3208
-
3209
- def test_bug_expr_dot_fid_mod
3210
- assert_scanned("foo.x!if 1",
3211
- :tIDENTIFIER, 'foo', [0, 3],
3212
- :tDOT, '.', [3, 4],
3213
- :tFID, 'x!', [4, 6],
3214
- :kIF_MOD, 'if', [6, 8],
3215
- :tINTEGER, 1, [9, 10])
3216
- end
3217
-
3218
- def test_bug_expr_mid_comment
3219
- assert_scanned("rescue #bar\nprint",
3220
- :kRESCUE, 'rescue', [0, 6],
3221
- :tNL, nil, [11, 12],
3222
- :tIDENTIFIER, 'print', [12, 17])
3223
- end
3224
-
3225
- def test_bug_expr_mid_bareword
3226
- assert_scanned("begin; rescue rescue1",
3227
- :kBEGIN, 'begin', [0, 5],
3228
- :tSEMI, ';', [5, 6],
3229
- :kRESCUE, 'rescue', [7, 13],
3230
- :tIDENTIFIER, 'rescue1', [14, 21])
3231
- end
3232
-
3233
- def test_bug_expr_value_document
3234
- assert_scanned("1;\n=begin\n=end",
3235
- :tINTEGER, 1, [0, 1],
3236
- :tSEMI, ';', [1, 2])
3237
- end
3238
-
3239
- def test_bug_expr_end_colon
3240
- assert_scanned("'foo':'bar'",
3241
- :tSTRING, 'foo', [0, 5],
3242
- :tCOLON, ':', [5, 6],
3243
- :tSTRING, 'bar', [6, 11])
3244
- end
3245
-
3246
- def test_bug_expr_value_rescue_colon2
3247
- @lex.state = :expr_value
3248
- assert_scanned("rescue::Exception",
3249
- :kRESCUE, 'rescue', [0, 6],
3250
- :tCOLON3, '::', [6, 8],
3251
- :tCONSTANT, 'Exception', [8, 17])
3252
- end
3253
-
3254
- def test_bug_expr_endarg_braces
3255
- assert_scanned("let [] {",
3256
- :tIDENTIFIER, 'let', [0, 3],
3257
- :tLBRACK, '[', [4, 5],
3258
- :tRBRACK, ']', [5, 6],
3259
- :tLBRACE_ARG, '{', [7, 8])
3260
- end
3261
-
3262
- def test_bug_line_begin_label
3263
- setup_lexer(19)
3264
- assert_scanned("foo:bar",
3265
- :tIDENTIFIER, 'foo', [0, 3],
3266
- :tSYMBOL, 'bar', [3, 7])
3267
- end
3268
-
3269
- def test_bug_interp_expr_value
3270
- assert_scanned('"#{f:a}"',
3271
- :tSTRING_BEG, '"', [0, 1],
3272
- :tSTRING_DBEG, '#{', [1, 3],
3273
- :tIDENTIFIER, 'f', [3, 4],
3274
- :tSYMBOL, 'a', [4, 6],
3275
- :tRCURLY, '}', [6, 7],
3276
- :tSTRING_END, '"', [7, 8])
3277
- end
3278
-
3279
- def test_bug_const_e
3280
- assert_scanned('E10',
3281
- :tCONSTANT, 'E10', [0, 3])
3282
- assert_scanned('E4U',
3283
- :tCONSTANT, 'E4U', [0, 3])
3284
- end
3285
-
3286
- def test_bug_symbol_newline
3287
- assert_scanned(":foo\n",
3288
- :tSYMBOL, 'foo', [0, 4],
3289
- :tNL, nil, [4, 5])
3290
-
3291
- assert_scanned(":foo=\n",
3292
- :tSYMBOL, 'foo=', [0, 5],
3293
- :tNL, nil, [5, 6])
3294
- end
3295
-
3296
- def test_bug_interleaved_heredoc
3297
- assert_scanned(%Q{<<w; "\nfoo\nw\n"},
3298
- :tSTRING_BEG, '<<"', [0, 3],
3299
- :tSTRING_CONTENT, "foo\n", [7, 11],
3300
- :tSTRING_END, 'w', [11, 12],
3301
- :tSEMI, ';', [3, 4],
3302
- :tSTRING_BEG, '"', [5, 6],
3303
- :tSTRING_CONTENT, "\n", [6, 7],
3304
- :tSTRING_END, '"', [13, 14])
3305
-
3306
- @lex.state = :expr_beg
3307
- assert_scanned(%Q{<<w; %w[\nfoo\nw\n1]},
3308
- :tSTRING_BEG, '<<"', [0, 3],
3309
- :tSTRING_CONTENT, "foo\n", [9, 13],
3310
- :tSTRING_END, 'w', [13, 14],
3311
- :tSEMI, ';', [3, 4],
3312
- :tQWORDS_BEG, '%w[', [5, 8],
3313
- :tSTRING_CONTENT, "1", [15, 16],
3314
- :tSPACE, nil, [16, 16],
3315
- :tSTRING_END, ']', [16, 17])
3316
-
3317
- @lex.state = :expr_beg
3318
- assert_scanned(%Q{<<w; "\#{\nfoo\nw\n}"},
3319
- :tSTRING_BEG, '<<"', [0, 3],
3320
- :tSTRING_CONTENT, "foo\n", [9, 13],
3321
- :tSTRING_END, 'w', [13, 14],
3322
- :tSEMI, ';', [3, 4],
3323
- :tSTRING_BEG, '"', [5, 6],
3324
- :tSTRING_DBEG, '#{', [6, 8],
3325
- :tRCURLY, '}', [15, 16],
3326
- :tSTRING_END, '"', [16, 17])
3327
- end
3328
-
3329
- def test_bug_fid_char
3330
- setup_lexer(19)
3331
- assert_scanned('eof??a',
3332
- :tFID, 'eof?', [0, 4],
3333
- :tCHARACTER, 'a', [4, 6])
3334
- end
3335
-
3336
- def test_bug_nonlabel_context__18
3337
- env = Parser::StaticEnvironment.new
3338
- env.declare "a"
3339
-
3340
- @lex.static_env = env
3341
- assert_scanned("1+a:a",
3342
- :tINTEGER, 1, [0, 1],
3343
- :tPLUS, '+', [1, 2],
3344
- :tIDENTIFIER, 'a', [2, 3],
3345
- :tCOLON, ':', [3, 4],
3346
- :tIDENTIFIER, 'a', [4, 5])
3347
- end
3348
-
3349
- def test_bug_string_percent_newline
3350
- assert_scanned(%Q{%\nfoo\n},
3351
- :tSTRING_BEG, "%\n", [0, 2],
3352
- :tSTRING_CONTENT, 'foo', [2, 5],
3353
- :tSTRING_END, "\n", [5, 6])
3354
- end
3355
-
3356
- def test_bug_string_percent_zero
3357
- assert_scanned(%Q{%\0foo\0},
3358
- :tSTRING_BEG, "%\0", [0, 2],
3359
- :tSTRING_CONTENT, 'foo', [2, 5],
3360
- :tSTRING_END, "\0", [5, 6])
3361
- end
3362
-
3363
- def test_bug_string_utf_escape_composition
3364
- assert_scanned(%q{"\xE2\x80\x99"},
3365
- :tSTRING, "\xE2\x80\x99", [0, 14])
3366
-
3367
- assert_scanned(utf(%q{"\xE2\x80\x99"}),
3368
- :tSTRING, utf('’'), [0, 14])
3369
- assert_scanned(utf(%q{"\342\200\231"}),
3370
- :tSTRING, utf('’'), [0, 14])
3371
- assert_scanned(utf(%q{"\M-b\C-\M-@\C-\M-Y"}),
3372
- :tSTRING, utf('’'), [0, 20])
3373
- end
3374
-
3375
- def test_bug_string_utf_escape_noop
3376
- assert_scanned(utf(%q{"\あ"}),
3377
- :tSTRING, utf("あ"), [0, 4])
3378
- end
3379
-
3380
- def test_bug_string_non_utf
3381
- assert_scanned(%Q{"caf\xE9"},
3382
- :tSTRING, "caf\xE9", [0, 6])
3383
- assert_scanned(%Q{"caf\xC3\xA9"},
3384
- :tSTRING, "caf\xC3\xA9", [0, 7])
3385
-
3386
- assert_scanned(utf(%q{"café"}),
3387
- :tSTRING, utf("café"), [0, 6])
3388
- end
3389
-
3390
- def test_bug_semi__END__
3391
- assert_scanned(%Q{foo;\n__END__},
3392
- :tIDENTIFIER, 'foo', [0, 3],
3393
- :tSEMI, ';', [3, 4])
3394
- end
3395
-
3396
- def test_bug_eql_end
3397
- assert_scanned(%Q{=begin\n#=end\n=end})
3398
- end
3399
-
3400
- def test_bug_hidden_eof
3401
- @lex.state = :expr_beg
3402
- assert_scanned(%Q{"foo\0\x1a\x04bar"},
3403
- :tSTRING_BEG, '"', [0, 1],
3404
- :tSTRING_CONTENT, "foo\0", [1, 5],
3405
- :tSTRING_CONTENT, "\x1a", [5, 6],
3406
- :tSTRING_CONTENT, "\x04", [6, 7],
3407
- :tSTRING_CONTENT, "bar", [7, 10],
3408
- :tSTRING_END, '"', [10, 11])
3409
-
3410
- @lex.state = :expr_beg
3411
- assert_scanned(%Q{'foo\0\x1a\x04bar'},
3412
- :tSTRING_BEG, "'", [0, 1],
3413
- :tSTRING_CONTENT, "foo\0", [1, 5],
3414
- :tSTRING_CONTENT, "\x1a", [5, 6],
3415
- :tSTRING_CONTENT, "\x04", [6, 7],
3416
- :tSTRING_CONTENT, "bar", [7, 10],
3417
- :tSTRING_END, "'", [10, 11])
3418
-
3419
- @lex.state = :expr_beg
3420
- assert_scanned(%Q{%w[foo\0\x1a\x04bar]},
3421
- :tQWORDS_BEG, '%w[', [0, 3],
3422
- :tSTRING_CONTENT, "foo\0", [3, 7],
3423
- :tSTRING_CONTENT, "\x1a", [7, 8],
3424
- :tSTRING_CONTENT, "\x04", [8, 9],
3425
- :tSTRING_CONTENT, "bar", [9, 12],
3426
- :tSPACE, nil, [12, 12],
3427
- :tSTRING_END, ']', [12, 13])
3428
-
3429
- @lex.state = :expr_beg
3430
- assert_scanned(%Q{%W[foo\0\x1a\x04bar]},
3431
- :tWORDS_BEG, '%W[', [0, 3],
3432
- :tSTRING_CONTENT, "foo\0", [3, 7],
3433
- :tSTRING_CONTENT, "\x1a", [7, 8],
3434
- :tSTRING_CONTENT, "\x04", [8, 9],
3435
- :tSTRING_CONTENT, "bar", [9, 12],
3436
- :tSPACE, nil, [12, 12],
3437
- :tSTRING_END, ']', [12, 13])
3438
-
3439
- @lex.state = :expr_beg
3440
- assert_scanned(%Q{# foo\0\nbar},
3441
- :tIDENTIFIER, 'bar', [7, 10])
3442
-
3443
- @lex.state = :line_begin
3444
- assert_scanned(%Q{=begin\n\0\n=end\nbar},
3445
- :tIDENTIFIER, 'bar', [14, 17])
3446
- end
3447
-
3448
- def test_bug_num_adj_kw
3449
- assert_scanned('1if',
3450
- :tINTEGER, 1, [0, 1],
3451
- :kIF_MOD, 'if', [1, 3])
3452
-
3453
- assert_scanned('1.0if',
3454
- :tFLOAT, 1.0, [0, 3],
3455
- :kIF_MOD, 'if', [3, 5])
3456
- end
3457
-
3458
- def test_bug_unicode_in_literal
3459
- setup_lexer(19)
3460
- assert_scanned('"\u00a4"',
3461
- :tSTRING, "\u00a4", [0, 8])
3462
- end
3463
-
3464
- def test_bug_utf32le_leak
3465
- setup_lexer(19)
3466
- @lex.force_utf32 = true
3467
- assert_scanned('"F0"',
3468
- :tSTRING, "F0", [0, 4])
3469
- end
3470
-
3471
- def test_bug_ragel_stack
3472
- assert_scanned("\"\#{$2 ? $2 : 1}\"",
3473
- :tSTRING_BEG, "\"", [0, 1],
3474
- :tSTRING_DBEG, "\#{", [1, 3],
3475
- :tNTH_REF, 2, [3, 5],
3476
- :tEH, "?", [6, 7],
3477
- :tNTH_REF, 2, [8, 10],
3478
- :tCOLON, ":", [11, 12],
3479
- :tINTEGER, 1, [13, 14],
3480
- :tRCURLY, "}", [14, 15],
3481
- :tSTRING_END, "\"", [15, 16])
3482
- end
3483
-
3484
- def test_bug_423
3485
- @lex.state = :expr_beg
3486
- assert_scanned(':&&',
3487
- :tSYMBEG, ':', [0, 1],
3488
- :tANDOP, '&&', [1, 3])
3489
-
3490
- @lex.state = :expr_beg
3491
- assert_scanned(':||',
3492
- :tSYMBEG, ':', [0, 1],
3493
- :tOROP, '||', [1, 3])
3494
- end
3495
-
3496
- def test_bug_418
3497
- setup_lexer 19
3498
-
3499
- assert_scanned("{\n=begin\nx: 1,\n=end\ny: 2}",
3500
- :tLBRACE, '{', [0, 1],
3501
- :tLABEL, 'y', [20, 22],
3502
- :tINTEGER, 2, [23, 24],
3503
- :tRCURLY, '}', [24, 25])
3504
- end
3505
-
3506
- def test_bug_407
3507
- setup_lexer(21)
3508
-
3509
- assert_scanned('123if cond',
3510
- :tINTEGER, 123, [0, 3],
3511
- :kIF_MOD, 'if', [3, 5],
3512
- :tIDENTIFIER, 'cond', [6, 10])
3513
-
3514
- assert_scanned('1.23if cond',
3515
- :tFLOAT, 1.23, [0, 4],
3516
- :kIF_MOD, 'if', [4, 6],
3517
- :tIDENTIFIER, 'cond', [7, 11])
3518
-
3519
- assert_scanned('123rescue cond',
3520
- :tINTEGER, 123, [0, 3],
3521
- :kRESCUE_MOD, 'rescue', [3, 9],
3522
- :tIDENTIFIER, 'cond', [10, 14])
3523
-
3524
- assert_scanned('1.23rescue cond',
3525
- :tFLOAT, 1.23, [0, 4],
3526
- :kRESCUE_MOD, 'rescue', [4, 10],
3527
- :tIDENTIFIER, 'cond', [11, 15])
3528
- end
3529
-
3530
- def test_parser_bug_486
3531
- setup_lexer(19)
3532
- assert_scanned(':!@',
3533
- :tSYMBOL, '!', [0, 3])
3534
-
3535
- setup_lexer(19)
3536
- assert_scanned(':~@',
3537
- :tSYMBOL, '~', [0, 3])
3538
- end
3539
-
3540
- def test_slash_only_in_heredocs
3541
- setup_lexer(23)
3542
- refute_scanned(%Q{<<~E\n\\\nE})
3543
-
3544
- setup_lexer(23)
3545
- refute_scanned(%Q{<<-E\n\\\nE})
3546
- end
3547
-
3548
- def test_escapes_in_squiggly_heredoc
3549
- setup_lexer(23)
3550
-
3551
- assert_scanned(%Q{<<~E\n\a\b\e\f\r\t\\\v\nE},
3552
- :tSTRING_BEG, '<<"', [0, 4],
3553
- :tSTRING_CONTENT, "\a\b\e\f\r\t\v\n", [5, 14],
3554
- :tSTRING_END, 'E', [14, 15],
3555
- :tNL, nil, [4, 5])
3556
-
3557
- setup_lexer(23)
3558
-
3559
- assert_scanned(%Q{<<-E\n\a\b\e\f\r\t\\\v\nE},
3560
- :tSTRING_BEG, '<<"', [0, 4],
3561
- :tSTRING_CONTENT, "\a\b\e\f\r\t\v\n", [5, 14],
3562
- :tSTRING_END, 'E', [14, 15],
3563
- :tNL, nil, [4, 5])
3564
- end
3565
-
3566
- def test_ambiguous_integer_re
3567
- assert_scanned('1re',
3568
- :tINTEGER, 1, [0, 1],
3569
- :tIDENTIFIER, 're', [1, 3])
3570
- end
3571
-
3572
- def lex_numbered_parameter(input)
3573
- @lex.max_numparam_stack.push
3574
-
3575
- @lex.context = Parser::Context.new
3576
- @lex.context.push(:block)
3577
-
3578
- source_buffer = Parser::Source::Buffer.new('(assert_lex_numbered_parameter)')
3579
- source_buffer.source = input
3580
-
3581
- @lex.source_buffer = source_buffer
3582
-
3583
- @lex.advance
3584
- end
3585
-
3586
- def assert_scanned_numbered_parameter(input)
3587
- lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
3588
-
3589
- assert_equal(lex_token, :tNUMPARAM)
3590
- assert_equal(lex_value, input.tr('@', ''))
3591
- assert_equal(lex_range.begin_pos, 0)
3592
- assert_equal(lex_range.end_pos, input.length)
3593
- end
3594
-
3595
- def refute_scanned_numbered_parameter(input, message = nil)
3596
- err = assert_raises Parser::SyntaxError do
3597
- lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
3598
- end
3599
-
3600
- if message
3601
- assert_equal(err.message, Parser::MESSAGES[message])
3602
-
3603
- assert_equal(err.diagnostic.location.begin_pos, 0)
3604
- assert_equal(err.diagnostic.location.end_pos, input.length)
3605
- end
3606
- end
3607
-
3608
- end