parser 2.7.1.1 → 3.0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -0
  3. data/lib/parser/all.rb +2 -0
  4. data/lib/parser/ast/processor.rb +5 -0
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +263 -23
  7. data/lib/parser/context.rb +5 -0
  8. data/lib/parser/current.rb +24 -6
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/diagnostic.rb +1 -1
  11. data/lib/parser/diagnostic/engine.rb +1 -2
  12. data/lib/parser/lexer.rb +887 -803
  13. data/lib/parser/macruby.rb +2214 -2189
  14. data/lib/parser/max_numparam_stack.rb +13 -5
  15. data/lib/parser/messages.rb +18 -0
  16. data/lib/parser/meta.rb +6 -5
  17. data/lib/parser/ruby18.rb +9 -3
  18. data/lib/parser/ruby19.rb +2297 -2289
  19. data/lib/parser/ruby20.rb +2413 -2397
  20. data/lib/parser/ruby21.rb +2419 -2411
  21. data/lib/parser/ruby22.rb +2468 -2460
  22. data/lib/parser/ruby23.rb +2452 -2452
  23. data/lib/parser/ruby24.rb +2435 -2430
  24. data/lib/parser/ruby25.rb +2220 -2214
  25. data/lib/parser/ruby26.rb +2220 -2214
  26. data/lib/parser/ruby27.rb +3715 -3615
  27. data/lib/parser/ruby28.rb +8047 -0
  28. data/lib/parser/ruby30.rb +8060 -0
  29. data/lib/parser/ruby31.rb +8226 -0
  30. data/lib/parser/rubymotion.rb +2190 -2182
  31. data/lib/parser/runner.rb +31 -2
  32. data/lib/parser/runner/ruby_rewrite.rb +2 -2
  33. data/lib/parser/source/buffer.rb +53 -28
  34. data/lib/parser/source/comment.rb +14 -1
  35. data/lib/parser/source/comment/associator.rb +31 -8
  36. data/lib/parser/source/map/method_definition.rb +25 -0
  37. data/lib/parser/source/range.rb +10 -3
  38. data/lib/parser/source/tree_rewriter.rb +100 -10
  39. data/lib/parser/source/tree_rewriter/action.rb +114 -21
  40. data/lib/parser/static_environment.rb +4 -0
  41. data/lib/parser/tree_rewriter.rb +1 -2
  42. data/lib/parser/variables_stack.rb +4 -0
  43. data/lib/parser/version.rb +1 -1
  44. data/parser.gemspec +3 -18
  45. metadata +17 -98
  46. data/.gitignore +0 -33
  47. data/.travis.yml +0 -42
  48. data/.yardopts +0 -21
  49. data/CHANGELOG.md +0 -1075
  50. data/CONTRIBUTING.md +0 -17
  51. data/Gemfile +0 -10
  52. data/README.md +0 -309
  53. data/Rakefile +0 -166
  54. data/ci/run_rubocop_specs +0 -14
  55. data/doc/AST_FORMAT.md +0 -2180
  56. data/doc/CUSTOMIZATION.md +0 -37
  57. data/doc/INTERNALS.md +0 -21
  58. data/doc/css/.gitkeep +0 -0
  59. data/doc/css/common.css +0 -68
  60. data/lib/parser/lexer.rl +0 -2536
  61. data/lib/parser/macruby.y +0 -2198
  62. data/lib/parser/ruby18.y +0 -1934
  63. data/lib/parser/ruby19.y +0 -2175
  64. data/lib/parser/ruby20.y +0 -2353
  65. data/lib/parser/ruby21.y +0 -2357
  66. data/lib/parser/ruby22.y +0 -2364
  67. data/lib/parser/ruby23.y +0 -2370
  68. data/lib/parser/ruby24.y +0 -2408
  69. data/lib/parser/ruby25.y +0 -2405
  70. data/lib/parser/ruby26.y +0 -2413
  71. data/lib/parser/ruby27.y +0 -2941
  72. data/lib/parser/rubymotion.y +0 -2182
  73. data/test/bug_163/fixtures/input.rb +0 -5
  74. data/test/bug_163/fixtures/output.rb +0 -5
  75. data/test/bug_163/rewriter.rb +0 -20
  76. data/test/helper.rb +0 -60
  77. data/test/parse_helper.rb +0 -319
  78. data/test/racc_coverage_helper.rb +0 -133
  79. data/test/test_base.rb +0 -31
  80. data/test/test_current.rb +0 -29
  81. data/test/test_diagnostic.rb +0 -96
  82. data/test/test_diagnostic_engine.rb +0 -62
  83. data/test/test_encoding.rb +0 -99
  84. data/test/test_lexer.rb +0 -3608
  85. data/test/test_lexer_stack_state.rb +0 -78
  86. data/test/test_parse_helper.rb +0 -80
  87. data/test/test_parser.rb +0 -9430
  88. data/test/test_runner_parse.rb +0 -35
  89. data/test/test_runner_rewrite.rb +0 -47
  90. data/test/test_source_buffer.rb +0 -162
  91. data/test/test_source_comment.rb +0 -36
  92. data/test/test_source_comment_associator.rb +0 -367
  93. data/test/test_source_map.rb +0 -15
  94. data/test/test_source_range.rb +0 -187
  95. data/test/test_source_rewriter.rb +0 -541
  96. data/test/test_source_rewriter_action.rb +0 -46
  97. data/test/test_source_tree_rewriter.rb +0 -253
  98. data/test/test_static_environment.rb +0 -45
  99. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  100. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  101. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
data/test/test_base.rb DELETED
@@ -1,31 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
- require 'parser/current'
5
-
6
- class TestBase < Minitest::Test
7
- include AST::Sexp
8
-
9
- def test_parse
10
- ast = Parser::CurrentRuby.parse('1')
11
- assert_equal s(:int, 1), ast
12
- end
13
-
14
- def test_parse_with_comments
15
- ast, comments = Parser::CurrentRuby.parse_with_comments('1 # foo')
16
- assert_equal s(:int, 1), ast
17
- assert_equal 1, comments.size
18
- assert_equal '# foo', comments.first.text
19
- end
20
-
21
- def test_loc_to_node
22
- ast = Parser::CurrentRuby.parse('1')
23
- assert_equal ast.loc.node, ast
24
- end
25
-
26
- def test_loc_dup
27
- ast = Parser::CurrentRuby.parse('1')
28
- assert_nil ast.loc.dup.node
29
- Parser::AST::Node.new(:root, [], :location => ast.loc)
30
- end
31
- end
data/test/test_current.rb DELETED
@@ -1,29 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
- require 'parser/current'
5
-
6
- class TestCurrent < Minitest::Test
7
- def test_current
8
- case RUBY_VERSION
9
- when '2.0.0'
10
- assert_equal Parser::Ruby20, Parser::CurrentRuby
11
- when /^2\.1\.\d+/
12
- assert_equal Parser::Ruby21, Parser::CurrentRuby
13
- when /^2\.2\.\d+/
14
- assert_equal Parser::Ruby22, Parser::CurrentRuby
15
- when /^2\.3\.\d+/
16
- assert_equal Parser::Ruby23, Parser::CurrentRuby
17
- when /^2\.4\.\d+/
18
- assert_equal Parser::Ruby24, Parser::CurrentRuby
19
- when /^2\.5\.\d+/
20
- assert_equal Parser::Ruby25, Parser::CurrentRuby
21
- when /^2\.6\.\d+/
22
- assert_equal Parser::Ruby26, Parser::CurrentRuby
23
- when /^2\.7\.\d+/
24
- assert_equal Parser::Ruby27, Parser::CurrentRuby
25
- else
26
- flunk "Update test_current for #{RUBY_VERSION}"
27
- end
28
- end
29
- end
@@ -1,96 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
-
5
- class TestDiagnostic < Minitest::Test
6
- def setup
7
- @buffer = Parser::Source::Buffer.new('(string)')
8
- @buffer.source = 'if (this is some bad code + bugs)'
9
-
10
- @range1 = Parser::Source::Range.new(@buffer, 0, 2) # if
11
- @range2 = Parser::Source::Range.new(@buffer, 4, 8) # this
12
- end
13
-
14
- def test_verifies_levels
15
- error = assert_raises ArgumentError do
16
- Parser::Diagnostic.new(:foobar, :escape_eof, {}, @range1)
17
- end
18
-
19
- assert_match /level/, error.message
20
- end
21
-
22
- def test_freezes
23
- string = 'foo'.dup
24
- highlights = [@range2]
25
-
26
- diag = Parser::Diagnostic.new(:error, :escape_eof, @range1, highlights)
27
- assert diag.frozen?
28
- assert diag.arguments.frozen?
29
- assert diag.highlights.frozen?
30
-
31
- refute string.frozen?
32
- refute highlights.frozen?
33
- end
34
-
35
- def test_render
36
- location = Parser::Source::Range.new(@buffer, 26, 27)
37
-
38
- highlights = [
39
- Parser::Source::Range.new(@buffer, 21, 25),
40
- Parser::Source::Range.new(@buffer, 28, 32)
41
- ]
42
-
43
- diag = Parser::Diagnostic.new(:error, :unexpected, { :character => '+' },
44
- location, highlights)
45
- assert_equal([
46
- "(string):1:27: error: unexpected `+'",
47
- '(string):1: if (this is some bad code + bugs)',
48
- '(string):1: ~~~~ ^ ~~~~ '
49
- ], diag.render)
50
- end
51
-
52
- def test_multiline_render
53
- @buffer = Parser::Source::Buffer.new('(string)')
54
- @buffer.source = "abc abc abc\ndef def def\nghi ghi ghi\n"
55
-
56
- location = Parser::Source::Range.new(@buffer, 4, 27)
57
-
58
- highlights = [
59
- Parser::Source::Range.new(@buffer, 0, 3),
60
- Parser::Source::Range.new(@buffer, 28, 31)
61
- ]
62
-
63
- diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'ghi' },
64
- location, highlights)
65
-
66
- assert_equal([
67
- "(string):1:5-3:3: error: unexpected token ghi",
68
- '(string):1: abc abc abc',
69
- '(string):1: ~~~ ^~~~~~~...',
70
- '(string):3: ghi ghi ghi',
71
- '(string):3: ~~~ ~~~ '
72
- ], diag.render)
73
- end
74
-
75
- def test_bug_error_on_newline
76
- # regression test; see GitHub issue 273
77
- source = <<-CODE
78
- {
79
- foo: ->() # I forgot my brace
80
- }
81
- }
82
- CODE
83
- @buffer = Parser::Source::Buffer.new('(string)')
84
- @buffer.source = source
85
-
86
- location = Parser::Source::Range.new(@buffer, 33, 34)
87
- diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'tNL' },
88
- location)
89
-
90
- assert_equal([
91
- '(string):2:32: error: unexpected token tNL',
92
- '(string):2: foo: ->() # I forgot my brace',
93
- '(string):2: ^'
94
- ], diag.render)
95
- end
96
- end
@@ -1,62 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'helper'
4
-
5
- class TestDiagnosticEngine < Minitest::Test
6
- def setup
7
- @buffer = Parser::Source::Buffer.new('(source)')
8
- @buffer.source = 'foobar'
9
-
10
- @engine = Parser::Diagnostic::Engine.new
11
-
12
- @queue = []
13
- @engine.consumer = lambda { |diag| @queue << diag }
14
- end
15
-
16
- def test_process_warnings
17
- warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
18
- @engine.process(warn)
19
-
20
- assert_equal [warn], @queue
21
- end
22
-
23
- def test_ignore_warnings
24
- @engine.ignore_warnings = true
25
-
26
- warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
27
- @engine.process(warn)
28
-
29
- assert_equal [], @queue
30
- end
31
-
32
- def test_all_errors_are_fatal
33
- @engine.all_errors_are_fatal = true
34
-
35
- error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
36
-
37
- err = assert_raises Parser::SyntaxError do
38
- @engine.process(error)
39
- end
40
-
41
- assert_equal error, err.diagnostic
42
-
43
- assert_equal [error], @queue
44
- end
45
-
46
- def test_all_errors_are_collected
47
- error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
48
- @engine.process(error)
49
-
50
- assert_equal [error], @queue
51
- end
52
-
53
- def test_fatal_error
54
- fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, @buffer, 1..2)
55
-
56
- assert_raises Parser::SyntaxError do
57
- @engine.process(fatal)
58
- end
59
-
60
- assert_equal [fatal], @queue
61
- end
62
- end
@@ -1,99 +0,0 @@
1
- # encoding: binary
2
- # frozen_string_literal: true
3
-
4
- require 'helper'
5
-
6
- class TestEncoding < Minitest::Test
7
- include AST::Sexp
8
-
9
- def recognize(string)
10
- Parser::Source::Buffer.recognize_encoding(string)
11
- end
12
-
13
- require 'parser/all'
14
-
15
- def test_default
16
- assert_nil recognize('foobar')
17
- end
18
-
19
- def test_bom
20
- assert_equal Encoding::UTF_8, recognize("\xef\xbb\xbf\nfoobar")
21
- assert_equal Encoding::UTF_8, recognize("\xef\xbb\xbf# coding:koi8-r\nfoobar")
22
- end
23
-
24
- def test_magic_comment
25
- assert_equal Encoding::KOI8_R, recognize("# coding:koi8-r\nfoobar")
26
- end
27
-
28
- def test_shebang
29
- assert_equal Encoding::KOI8_R, recognize("#!/bin/foo\n# coding:koi8-r\nfoobar")
30
- assert_nil recognize("#!/bin/foo\n")
31
- end
32
-
33
- def test_case
34
- assert_equal Encoding::KOI8_R, recognize("# coding:KoI8-r\nfoobar")
35
- end
36
-
37
- def test_space
38
- assert_equal Encoding::KOI8_R, recognize("# coding : koi8-r\nfoobar")
39
- end
40
-
41
- def test_empty
42
- assert_nil recognize('')
43
- end
44
-
45
- def test_no_comment
46
- assert_nil recognize(%{require 'cane/encoding_aware_iterator'})
47
- end
48
-
49
- def test_adjacent
50
- assert_nil recognize('# codingkoi8-r')
51
- assert_nil recognize('# coding koi8-r')
52
- end
53
-
54
- def test_utf8_mac
55
- assert_equal Encoding::UTF8_MAC, recognize('# coding: utf8-mac')
56
- end
57
-
58
- def test_suffix
59
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-dos')
60
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-unix')
61
- assert_equal Encoding::UTF_8, recognize('# coding: utf-8-mac')
62
-
63
- assert_raises(ArgumentError) do
64
- assert_nil recognize('# coding: utf-8-dicks')
65
- end
66
- end
67
-
68
- def test_parse_18_invalid_enc
69
- ast = Parser::Ruby18.parse("# encoding:feynman-diagram\n1")
70
- assert_equal ast, s(:int, 1)
71
- end
72
-
73
- def test_parse_19_invalid_enc
74
- assert_raises(ArgumentError) do
75
- Parser::Ruby19.parse("# encoding:feynman-diagram\n1")
76
- end
77
- end
78
-
79
- def test_ending_comment
80
- assert_nil recognize('foo # coding: koi8-r')
81
- end
82
-
83
- def test_wrong_prefix
84
- assert_nil recognize('# decoding: koi8-r')
85
- end
86
-
87
- def test_no_spaces
88
- assert_equal Encoding::KOI8_R, recognize('#encoding:koi8-r')
89
- assert_equal Encoding::KOI8_R, recognize('#coding:koi8-r')
90
- end
91
-
92
- def test_underscore_and_star_characters
93
- assert_equal Encoding::KOI8_R, recognize('# -*- encoding: koi8-r -*-')
94
- end
95
-
96
- def test_garbage_around_encoding_comment
97
- assert_equal Encoding::KOI8_R, recognize('# 1$# -*- &)* encoding: koi8-r 1$# -*- &)*')
98
- end
99
- end
data/test/test_lexer.rb DELETED
@@ -1,3608 +0,0 @@
1
- # encoding: ascii-8bit
2
- # frozen_string_literal: true
3
-
4
- require 'helper'
5
- require 'complex'
6
-
7
- class TestLexer < Minitest::Test
8
- def setup_lexer(version)
9
- @lex = Parser::Lexer.new(version)
10
-
11
- @lex.comments = []
12
- @lex.diagnostics = Parser::Diagnostic::Engine.new
13
- @lex.diagnostics.all_errors_are_fatal = true
14
- # @lex.diagnostics.consumer = lambda { |diag| $stderr.puts "", diag.render }
15
- end
16
-
17
- def setup
18
- setup_lexer 18
19
- end
20
-
21
- #
22
- # Tools
23
- #
24
-
25
- def utf(str)
26
- str.dup.force_encoding(Encoding::UTF_8)
27
- end
28
-
29
- #
30
- # Additional matchers
31
- #
32
-
33
- def refute_scanned(s, *args)
34
- assert_raises Parser::SyntaxError do
35
- assert_scanned(s, *args)
36
- end
37
- end
38
-
39
- def assert_escape(expected, input)
40
- source_buffer = Parser::Source::Buffer.new('(assert_escape)')
41
-
42
- source_buffer.source = "\"\\#{input}\"".encode(input.encoding)
43
-
44
- @lex.reset
45
- @lex.source_buffer = source_buffer
46
-
47
- lex_token, (lex_value, *) = @lex.advance
48
-
49
- lex_value.force_encoding(Encoding::BINARY)
50
-
51
- assert_equal [:tSTRING, expected],
52
- [lex_token, lex_value],
53
- source_buffer.source
54
- end
55
-
56
- def refute_escape(input)
57
- err = assert_raises Parser::SyntaxError do
58
- @lex.state = :expr_beg
59
- assert_scanned "%Q[\\#{input}]"
60
- end
61
- assert_equal :fatal, err.diagnostic.level
62
- end
63
-
64
- def assert_lex_fname(name, type, range)
65
- begin_pos, end_pos = range
66
- assert_scanned("def #{name} ",
67
- :kDEF, 'def', [0, 3],
68
- type, name, [begin_pos + 4, end_pos + 4])
69
-
70
- assert_equal :expr_endfn, @lex.state
71
- end
72
-
73
- def assert_scanned(input, *args)
74
- source_buffer = Parser::Source::Buffer.new('(assert_scanned)')
75
- source_buffer.source = input
76
-
77
- @lex.reset(false)
78
- @lex.source_buffer = source_buffer
79
-
80
- until args.empty? do
81
- token, value, (begin_pos, end_pos) = args.shift(3)
82
-
83
- lex_token, (lex_value, lex_range) = @lex.advance
84
- assert lex_token, 'no more tokens'
85
- assert_operator [lex_token, lex_value], :eql?, [token, value], input
86
- assert_equal begin_pos, lex_range.begin_pos
87
- assert_equal end_pos, lex_range.end_pos
88
- end
89
-
90
- lex_token, (lex_value, *) = @lex.advance
91
- refute lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
92
- end
93
-
94
- #
95
- # Tests
96
- #
97
-
98
- def test_read_escape
99
- assert_escape "\\", "\\"
100
- assert_escape "\n", "n"
101
- assert_escape "\t", "t"
102
- assert_escape "\r", "r"
103
- assert_escape "\f", "f"
104
- assert_escape "\13", "v"
105
- assert_escape "\0", "0"
106
- assert_escape "\07", "a"
107
- assert_escape "\007", "a"
108
- assert_escape "\033", "e"
109
- assert_escape "\377", "377"
110
- assert_escape "\377", "xff"
111
- assert_escape "\010", "b"
112
- assert_escape " ", "s"
113
- assert_escape "q", "q" # plain vanilla escape
114
- end
115
-
116
- def test_read_escape_c
117
- assert_escape "\030", "C-x"
118
- assert_escape "\030", "cx"
119
- assert_escape "\230", 'C-\M-x'
120
- assert_escape "\230", 'c\M-x'
121
-
122
- assert_escape "\177", "C-?"
123
- assert_escape "\177", "c?"
124
- assert_escape "\r", "cM"
125
- end
126
-
127
- def test_read_escape_m
128
- assert_escape "\370", "M-x"
129
- assert_escape "\230", 'M-\C-x'
130
- assert_escape "\230", 'M-\cx'
131
- end
132
-
133
- def test_read_escape_errors
134
- refute_escape ""
135
-
136
- refute_escape "M"
137
- refute_escape "M-"
138
- refute_escape "Mx"
139
-
140
- refute_escape "Cx"
141
- refute_escape "C"
142
- refute_escape "C-"
143
-
144
- refute_escape "c"
145
-
146
- refute_escape "x"
147
- end
148
-
149
- def test_read_escape_unicode__19
150
- assert_escape "\x09", 'u{9}'
151
- assert_escape "\x31", 'u{31}'
152
- assert_escape "\x09\x01", 'u{9 1}'
153
-
154
- assert_escape "\xc4\xa3", utf('u0123')
155
- assert_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", utf('u{123 f0 B123}')
156
- end
157
-
158
- def test_read_escape_unicode_bad__19
159
- refute_escape 'u123'
160
- refute_escape 'u{}'
161
- refute_escape 'u{123 f0h}'
162
- refute_escape 'u{123 f0'
163
- end
164
-
165
- def test_read_escape_whitespaces__27
166
- setup_lexer 27
167
-
168
- [ *(0..8), *(14..31) ].each do |code|
169
- @lex.reset
170
- refute_scanned "\"\\C-" + code.chr + "\""
171
-
172
- @lex.reset
173
- refute_scanned "\"\\M-" + code.chr + "\""
174
-
175
- @lex.reset
176
- refute_scanned "\"\\C-\\M-" + code.chr + "\""
177
-
178
- @lex.reset
179
- refute_scanned "\"\\M-\\C-" + code.chr + "\""
180
- end
181
- end
182
-
183
- def test_ambiguous_uminus
184
- assert_scanned("m -3",
185
- :tIDENTIFIER, "m", [0, 1],
186
- :tUNARY_NUM, "-", [2, 3],
187
- :tINTEGER, 3, [3, 4])
188
- end
189
-
190
- def test_ambiguous_uplus
191
- assert_scanned("m +3",
192
- :tIDENTIFIER, "m", [0, 1],
193
- :tUNARY_NUM, "+", [2, 3],
194
- :tINTEGER, 3, [3, 4])
195
- end
196
-
197
- def test_and
198
- assert_scanned "&", :tAMPER, "&", [0, 1]
199
- end
200
-
201
- def test_and2
202
- @lex.state = :expr_end
203
-
204
- assert_scanned "&&", :tANDOP, "&&", [0, 2]
205
- end
206
-
207
- def test_and2_equals
208
- @lex.state = :expr_end
209
-
210
- assert_scanned "&&=", :tOP_ASGN, "&&", [0, 3]
211
- end
212
-
213
- def test_and_arg
214
- @lex.state = :expr_arg
215
-
216
- assert_scanned(" &y",
217
- :tAMPER, "&", [1, 2],
218
- :tIDENTIFIER, "y", [2, 3])
219
- end
220
-
221
- def test_and_equals
222
- @lex.state = :expr_end
223
-
224
- assert_scanned "&=", :tOP_ASGN, "&", [0, 2]
225
- end
226
-
227
- def test_and_expr
228
- @lex.state = :expr_arg
229
-
230
- assert_scanned("x & y",
231
- :tIDENTIFIER, "x", [0, 1],
232
- :tAMPER2, "&", [2, 3],
233
- :tIDENTIFIER, "y", [4, 5])
234
- end
235
-
236
- def test_and_meth
237
- assert_lex_fname "&", :tAMPER2, [0, 1]
238
- end
239
-
240
- def test_and_dot_arg
241
- @lex.state = :expr_arg
242
-
243
- assert_scanned "&.", :tANDDOT, "&.", [0, 2]
244
- end
245
-
246
- def test_and_dot_cmdarg
247
- @lex.state = :expr_cmdarg
248
-
249
- assert_scanned "&.", :tANDDOT, "&.", [0, 2]
250
- end
251
-
252
- def test_assoc
253
- assert_scanned "=>", :tASSOC, "=>", [0, 2]
254
- end
255
-
256
- def test_label__18
257
- assert_scanned("{a:b",
258
- :tLBRACE, "{", [0, 1],
259
- :tIDENTIFIER, "a", [1, 2],
260
- :tSYMBOL, "b", [2, 4])
261
- end
262
-
263
- def test_label_in_params__18
264
- assert_scanned("foo(a:b",
265
- :tIDENTIFIER, "foo", [0, 3],
266
- :tLPAREN2, "(", [3, 4],
267
- :tIDENTIFIER, "a", [4, 5],
268
- :tSYMBOL, "b", [5, 7])
269
- end
270
-
271
- def test_label__19
272
- setup_lexer 19
273
-
274
- assert_scanned("{a:b",
275
- :tLBRACE, "{", [0, 1],
276
- :tLABEL, "a", [1, 3],
277
- :tIDENTIFIER, "b", [3, 4])
278
- end
279
-
280
- def test_label_in_params__19
281
- setup_lexer 19
282
-
283
- assert_scanned("foo(a:b",
284
- :tIDENTIFIER, "foo", [0, 3],
285
- :tLPAREN2, "(", [3, 4],
286
- :tLABEL, "a", [4, 6],
287
- :tIDENTIFIER, "b", [6, 7])
288
- end
289
-
290
- def test_label_fid__19
291
- setup_lexer 19
292
-
293
- assert_scanned("{a?:true",
294
- :tLBRACE, '{', [0, 1],
295
- :tLABEL, 'a?', [1, 4],
296
- :kTRUE, 'true', [4, 8])
297
- end
298
-
299
- def test_label__22
300
- setup_lexer 22
301
-
302
- assert_scanned("{'a':",
303
- :tLBRACE, '{', [0, 1],
304
- :tSTRING_BEG, "'", [1, 2],
305
- :tSTRING_CONTENT, 'a', [2, 3],
306
- :tLABEL_END, "'", [3, 5])
307
- end
308
-
309
- def test_label_nested__22
310
- setup_lexer 22
311
-
312
- assert_scanned("{'a\":':",
313
- :tLBRACE, '{', [0, 1],
314
- :tSTRING_BEG, "'", [1, 2],
315
- :tSTRING_CONTENT, 'a":', [2, 5],
316
- :tLABEL_END, "'", [5, 7])
317
- end
318
-
319
- def test_label_colon2__22
320
- setup_lexer 22
321
-
322
- assert_scanned("{'a'::",
323
- :tLBRACE, '{', [0, 1],
324
- :tSTRING, "a", [1, 4],
325
- :tCOLON2, '::', [4, 6])
326
- end
327
-
328
- def test_pct_string_colon__22
329
- setup_lexer 22
330
-
331
- assert_scanned("{%'a':",
332
- :tLBRACE, '{', [0, 1],
333
- :tSTRING_BEG, "%'", [1, 3],
334
- :tSTRING_CONTENT, 'a', [3, 4],
335
- :tSTRING_END, "'", [4, 5],
336
- :tCOLON, ':', [5, 6])
337
- end
338
-
339
- def test_command_start__19
340
- setup_lexer 19
341
-
342
- %w[case elsif for in until when while
343
- if unless and or].each do |keyword|
344
- token = "k#{keyword.upcase}".to_sym
345
-
346
- @lex.reset
347
- assert_scanned("#{keyword} a:b",
348
- token, keyword, [0, keyword.length],
349
- :tIDENTIFIER, "a", [keyword.length + 1, keyword.length + 2],
350
- :tSYMBOL, "b", [keyword.length + 2, keyword.length + 4])
351
- end
352
- end
353
-
354
- def test_mod_not_command_start__19
355
- setup_lexer 19
356
-
357
- %w[if unless while until rescue].each do |keyword|
358
- token = "k#{keyword.upcase}_MOD".to_sym
359
-
360
- @lex.state = :expr_end
361
- assert_scanned("#{keyword} a:b",
362
- token, keyword, [0, keyword.length],
363
- :tLABEL, "a", [keyword.length + 1, keyword.length + 3],
364
- :tIDENTIFIER, "b", [keyword.length + 3, keyword.length + 4])
365
- end
366
- end
367
-
368
- def test_back_ref
369
- assert_scanned("[$&, $`, $', $+]",
370
- :tLBRACK, "[", [0, 1],
371
- :tBACK_REF, "$&", [1, 3], :tCOMMA, ",", [3, 4],
372
- :tBACK_REF, "$`", [5, 7], :tCOMMA, ",", [7, 8],
373
- :tBACK_REF, "$'", [9, 11], :tCOMMA, ",", [11, 12],
374
- :tBACK_REF, "$+", [13, 15],
375
- :tRBRACK, "]", [15, 16])
376
- end
377
-
378
- def test_backslash
379
- assert_scanned("1 \\\n+ 2",
380
- :tINTEGER, 1, [0, 1],
381
- :tPLUS, "+", [4, 5],
382
- :tINTEGER, 2, [6, 7])
383
- end
384
-
385
- def test_backslash_bad
386
- refute_scanned("1 \\ + 2",
387
- :tINTEGER, 1, [0, 1])
388
- end
389
-
390
- def test_backtick
391
- assert_scanned("`ls`",
392
- :tXSTRING_BEG, "`", [0, 1],
393
- :tSTRING_CONTENT, "ls", [1, 3],
394
- :tSTRING_END, "`", [3, 4])
395
- end
396
-
397
- def test_backtick_cmdarg
398
- @lex.state = :expr_dot
399
- assert_scanned("\n`", :tBACK_REF2, "`", [1, 2]) # \n ensures expr_cmd
400
-
401
- assert_equal :expr_arg, @lex.state
402
- end
403
-
404
- def test_backtick_dot
405
- @lex.state = :expr_dot
406
- assert_scanned("a.`(3)",
407
- :tIDENTIFIER, "a", [0, 1],
408
- :tDOT, ".", [1, 2],
409
- :tBACK_REF2, "`", [2, 3],
410
- :tLPAREN2, "(", [3, 4],
411
- :tINTEGER, 3, [4, 5],
412
- :tRPAREN, ")", [5, 6])
413
- end
414
-
415
- def test_backtick_method
416
- @lex.state = :expr_fname
417
- assert_scanned("`", :tBACK_REF2, "`", [0, 1])
418
- assert_equal :expr_endfn, @lex.state
419
- end
420
-
421
- def test_bad_char
422
- refute_scanned(" \010 ")
423
- end
424
-
425
- def test_bang
426
- assert_scanned "!", :tBANG, "!", [0, 1]
427
- end
428
-
429
- def test_bang_equals
430
- assert_scanned "!=", :tNEQ, "!=", [0, 2]
431
- end
432
-
433
- def test_bang_tilde
434
- assert_scanned "!~", :tNMATCH, "!~", [0, 2]
435
- end
436
-
437
- def test_def_ubang
438
- setup_lexer(20)
439
-
440
- @lex.state = :expr_fname
441
- assert_scanned '!@', :tBANG, '!@', [0, 2]
442
- end
443
-
444
- def test_carat
445
- assert_scanned "^", :tCARET, "^", [0, 1]
446
- end
447
-
448
- def test_carat_equals
449
- assert_scanned "^=", :tOP_ASGN, "^", [0, 2]
450
- end
451
-
452
- def test_colon2
453
- assert_scanned("A::B",
454
- :tCONSTANT, "A", [0, 1],
455
- :tCOLON2, "::", [1, 3],
456
- :tCONSTANT, "B", [3, 4])
457
-
458
- @lex.state = :expr_arg
459
- assert_scanned("::Array",
460
- :tCOLON2, "::", [0, 2],
461
- :tCONSTANT, "Array", [2, 7])
462
- end
463
-
464
- def test_colon3
465
- assert_scanned("::Array",
466
- :tCOLON3, "::", [0, 2],
467
- :tCONSTANT, "Array", [2, 7])
468
-
469
- @lex.state = :expr_arg
470
- assert_scanned(" ::Array",
471
- :tCOLON3, "::", [1, 3],
472
- :tCONSTANT, "Array", [3, 8])
473
- end
474
-
475
- def test_comma
476
- assert_scanned ",", :tCOMMA, ",", [0, 1]
477
- end
478
-
479
- def test_comment
480
- [26, 27].each do |version|
481
- setup_lexer(version)
482
-
483
- assert_scanned("1 # one\n# two\n2",
484
- :tINTEGER, 1, [0, 1],
485
- :tNL, nil, [7, 8],
486
- :tINTEGER, 2, [14, 15])
487
-
488
- assert_equal 2, @lex.comments.length
489
- assert_equal '# one', @lex.comments[0].text
490
- assert_equal '# two', @lex.comments[1].text
491
- end
492
- end
493
-
494
- def test_comment_expr_beg
495
- assert_scanned("{#1\n}",
496
- :tLBRACE, "{", [0, 1],
497
- :tRCURLY, "}", [4, 5])
498
- end
499
-
500
- def test_comment_begin
501
- assert_scanned("=begin\nblah\nblah\n=end\n42",
502
- :tINTEGER, 42, [22, 24])
503
- assert_equal 1, @lex.comments.length
504
- assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments[0].text
505
- end
506
-
507
- def test_comment_begin_bad
508
- refute_scanned("=begin\nblah\nblah\n")
509
- end
510
-
511
- def test_comment_begin_not_comment
512
- assert_scanned("beginfoo = 5\np x \\\n=beginfoo",
513
- :tIDENTIFIER, "beginfoo", [0, 8],
514
- :tEQL, "=", [9, 10],
515
- :tINTEGER, 5, [11, 12],
516
- :tNL, nil, [12, 13],
517
- :tIDENTIFIER, "p", [13, 14],
518
- :tIDENTIFIER, "x", [15, 16],
519
- :tEQL, "=", [19, 20],
520
- :tIDENTIFIER, "beginfoo", [20, 28])
521
- end
522
-
523
- def test_comment_begin_space
524
- assert_scanned("=begin blah\nblah\n=end\n")
525
-
526
- assert_equal 1, @lex.comments.length
527
- assert_equal "=begin blah\nblah\n=end\n", @lex.comments[0].text
528
- end
529
-
530
- def test_comment_end_space_and_text
531
- assert_scanned("=begin blah\nblah\n=end blab\n")
532
-
533
- assert_equal 1, @lex.comments.length
534
- assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments[0].text
535
- end
536
-
537
- def test_comment_eos
538
- assert_scanned("# comment")
539
- end
540
-
541
- def test_constant
542
- assert_scanned("ArgumentError",
543
- :tCONSTANT, "ArgumentError", [0, 13])
544
- end
545
-
546
- def test_constant_semi
547
- assert_scanned("ArgumentError;",
548
- :tCONSTANT, "ArgumentError", [0, 13],
549
- :tSEMI, ";", [13, 14])
550
- end
551
-
552
- def test_cvar
553
- assert_scanned "@@blah", :tCVAR, "@@blah", [0, 6]
554
- end
555
-
556
- def test_cvar_bad
557
- refute_scanned "@@1"
558
- end
559
-
560
- def test_div
561
- assert_scanned("a / 2",
562
- :tIDENTIFIER, "a", [0, 1],
563
- :tDIVIDE, "/", [2, 3],
564
- :tINTEGER, 2, [4, 5])
565
- end
566
-
567
- def test_div_equals
568
- assert_scanned("a /= 2",
569
- :tIDENTIFIER, "a", [0, 1],
570
- :tOP_ASGN, "/", [2, 4],
571
- :tINTEGER, 2, [5, 6])
572
- end
573
-
574
- def test_do
575
- assert_scanned("x do 42 end",
576
- :tIDENTIFIER, "x", [0, 1],
577
- :kDO, "do", [2, 4],
578
- :tINTEGER, 42, [5, 7],
579
- :kEND, "end", [8, 11])
580
- end
581
-
582
- def test_do_block
583
- @lex.state = :expr_endarg
584
-
585
- assert_scanned("do 42 end",
586
- :kDO_BLOCK, "do", [0, 2],
587
- :tINTEGER, 42, [3, 5],
588
- :kEND, "end", [6, 9])
589
- end
590
-
591
- def test_do_cond
592
- @lex.cond.push true
593
-
594
- assert_scanned("x do 42 end",
595
- :tIDENTIFIER, "x", [0, 1],
596
- :kDO_COND, "do", [2, 4],
597
- :tINTEGER, 42, [5, 7],
598
- :kEND, "end", [8, 11])
599
- end
600
-
601
- def test_dot
602
- assert_scanned ".", :tDOT, ".", [0, 1]
603
- end
604
-
605
- def test_dot2
606
- assert_scanned "..", :tDOT2, "..", [0, 2]
607
- end
608
-
609
- def test_dot3
610
- assert_scanned "...", :tDOT3, "...", [0, 3]
611
- end
612
-
613
- def test_equals
614
- assert_scanned "=", :tEQL, "=", [0, 1]
615
- end
616
-
617
- def test_equals2
618
- assert_scanned "==", :tEQ, "==", [0, 2]
619
- end
620
-
621
- def test_equals3
622
- assert_scanned "===", :tEQQ, "===", [0, 3]
623
- end
624
-
625
- def test_equals_tilde
626
- assert_scanned "=~", :tMATCH, "=~", [0, 2]
627
- end
628
-
629
- def test_float
630
- assert_scanned "1.0", :tFLOAT, 1.0, [0, 3]
631
- end
632
-
633
- def test_float_bad_no_underscores
634
- refute_scanned "1__0.0"
635
- end
636
-
637
- def test_float_bad_no_zero_leading
638
- refute_scanned ".0"
639
- end
640
-
641
- def test_float_bad_trailing_underscore
642
- refute_scanned "123_.0"
643
- end
644
-
645
- def test_float_call
646
- assert_scanned("1.0.to_s",
647
- :tFLOAT, 1.0, [0, 3],
648
- :tDOT, ".", [3, 4],
649
- :tIDENTIFIER, "to_s", [4, 8])
650
- end
651
-
652
- def test_float_dot_E
653
- assert_scanned "1.0E10", :tFLOAT, 1.0e10, [0, 6]
654
- end
655
-
656
- def test_float_dot_E_neg
657
- assert_scanned("-1.0E10",
658
- :tUNARY_NUM, "-", [0, 1],
659
- :tFLOAT, 1.0e10, [1, 7])
660
- end
661
-
662
- def test_float_dot_E_pos
663
- assert_scanned("+1.0E10",
664
- :tUNARY_NUM, "+", [0, 1],
665
- :tFLOAT, 1.0e10, [1, 7])
666
- end
667
-
668
- def test_float_dot_e
669
- assert_scanned "1.0e10", :tFLOAT, 1.0e10, [0, 6]
670
- end
671
-
672
- def test_float_dot_e_neg
673
- assert_scanned("-1.0e10",
674
- :tUNARY_NUM, "-", [0, 1],
675
- :tFLOAT, 1.0e10, [1, 7])
676
- end
677
-
678
- def test_float_dot_e_pos
679
- assert_scanned("+1.0e10",
680
- :tUNARY_NUM, "+", [0, 1],
681
- :tFLOAT, 1.0e10, [1, 7])
682
- end
683
-
684
- def test_float_e
685
- assert_scanned "1e10", :tFLOAT, 1e10, [0, 4]
686
- end
687
-
688
- def test_float_e_bad_trailing_underscore
689
- refute_scanned "123_e10"
690
- end
691
-
692
- def test_float_e_minus
693
- assert_scanned "1e-10", :tFLOAT, 1e-10, [0, 5]
694
- end
695
-
696
- def test_float_e_neg
697
- assert_scanned("-1e10",
698
- :tUNARY_NUM, "-", [0, 1],
699
- :tFLOAT, 1e10, [1, 5])
700
- end
701
-
702
- def test_float_e_neg_minus
703
- assert_scanned("-1e-10",
704
- :tUNARY_NUM, "-", [0, 1],
705
- :tFLOAT, 1e-10, [1, 6])
706
- end
707
-
708
- def test_float_e_neg_plus
709
- assert_scanned("-1e+10",
710
- :tUNARY_NUM, "-", [0, 1],
711
- :tFLOAT, 1e10, [1, 6])
712
- end
713
-
714
- def test_float_e_pos
715
- assert_scanned("+1e10",
716
- :tUNARY_NUM, "+", [0, 1],
717
- :tFLOAT, 1e10, [1, 5])
718
- end
719
-
720
- def test_float_e_pos_minus
721
- assert_scanned("+1e-10",
722
- :tUNARY_NUM, "+", [0, 1],
723
- :tFLOAT, 1e-10, [1, 6])
724
- end
725
-
726
- def test_float_e_pos_plus
727
- assert_scanned("+1e+10",
728
- :tUNARY_NUM, "+", [0, 1],
729
- :tFLOAT, 1e10, [1, 6])
730
- end
731
-
732
- def test_float_e_plus
733
- assert_scanned "1e+10", :tFLOAT, 1e10, [0, 5]
734
- end
735
-
736
- def test_float_e_zero
737
- assert_scanned "0e0", :tFLOAT, 0e0, [0, 3]
738
- end
739
-
740
- def test_float_e_nothing
741
- [18, 19, 20].each do |version|
742
- setup_lexer version
743
-
744
- refute_scanned "1end"
745
- refute_scanned "1.1end"
746
- end
747
-
748
- setup_lexer 21
749
-
750
- assert_scanned("1end",
751
- :tINTEGER, 1, [0, 1],
752
- :kEND, 'end', [1, 4])
753
- assert_scanned("1.1end",
754
- :tFLOAT, 1.1, [0, 3],
755
- :kEND, 'end', [3, 6])
756
- end
757
-
758
- def test_float_neg
759
- assert_scanned("-1.0",
760
- :tUNARY_NUM, "-", [0, 1],
761
- :tFLOAT, 1.0, [1, 4])
762
- end
763
-
764
- def test_float_pos
765
- assert_scanned("+1.0",
766
- :tUNARY_NUM, "+", [0, 1],
767
- :tFLOAT, 1.0, [1, 4])
768
- end
769
-
770
- def test_ge
771
- assert_scanned("a >= 2",
772
- :tIDENTIFIER, "a", [0, 1],
773
- :tGEQ, ">=", [2, 4],
774
- :tINTEGER, 2, [5, 6])
775
- end
776
-
777
- def test_global
778
- assert_scanned("$blah", :tGVAR, "$blah", [0, 5])
779
- end
780
-
781
- def test_global_backref
782
- assert_scanned("$`", :tBACK_REF, "$`", [0, 2])
783
- end
784
-
785
- # This was removed in 2.1.
786
- # def test_global_dash_nothing
787
- # assert_scanned("$- ", :tGVAR, "$-")
788
- # end
789
-
790
- def test_global_dash_something
791
- assert_scanned("$-x", :tGVAR, "$-x", [0, 3])
792
- end
793
-
794
- def test_global_number
795
- assert_scanned("$10", :tNTH_REF, 10, [0, 3])
796
- end
797
-
798
- def test_global_other
799
- assert_scanned("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
800
- :tLBRACK, "[", [0, 1],
801
- :tGVAR, "$~", [1, 3], :tCOMMA, ",", [3, 4],
802
- :tGVAR, "$*", [5, 7], :tCOMMA, ",", [7, 8],
803
- :tGVAR, "$$", [9, 11], :tCOMMA, ",", [11, 12],
804
- :tGVAR, "$\?", [13, 15], :tCOMMA, ",", [15, 16],
805
- :tGVAR, "$!", [17, 19], :tCOMMA, ",", [19, 20],
806
- :tGVAR, "$@", [21, 23], :tCOMMA, ",", [23, 24],
807
- :tGVAR, "$/", [25, 27], :tCOMMA, ",", [27, 28],
808
- :tGVAR, "$\\", [29, 31], :tCOMMA, ",", [31, 32],
809
- :tGVAR, "$;", [33, 35], :tCOMMA, ",", [35, 36],
810
- :tGVAR, "$,", [37, 39], :tCOMMA, ",", [39, 40],
811
- :tGVAR, "$.", [41, 43], :tCOMMA, ",", [43, 44],
812
- :tGVAR, "$=", [45, 47], :tCOMMA, ",", [47, 48],
813
- :tGVAR, "$:", [49, 51], :tCOMMA, ",", [51, 52],
814
- :tGVAR, "$<", [53, 55], :tCOMMA, ",", [55, 56],
815
- :tGVAR, "$>", [57, 59], :tCOMMA, ",", [59, 60],
816
- :tGVAR, "$\"", [61, 63],
817
- :tRBRACK, "]", [63, 64])
818
- end
819
-
820
- def test_global_underscore
821
- assert_scanned("$_",
822
- :tGVAR, "$_", [0, 2])
823
- end
824
-
825
- def test_global_weird
826
- assert_scanned("$__blah",
827
- :tGVAR, "$__blah", [0, 7])
828
- end
829
-
830
- def test_global_zero
831
- assert_scanned("$0", :tGVAR, "$0", [0, 2])
832
- end
833
-
834
- def test_gt
835
- assert_scanned("a > 2",
836
- :tIDENTIFIER, "a", [0, 1],
837
- :tGT, ">", [2, 3],
838
- :tINTEGER, 2, [4, 5])
839
- end
840
-
841
- def test_heredoc_backtick
842
- assert_scanned("a = <<`EOF`\n blah blah\nEOF\n",
843
- :tIDENTIFIER, "a", [0, 1],
844
- :tEQL, "=", [2, 3],
845
- :tXSTRING_BEG, "<<`", [4, 11],
846
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
847
- :tSTRING_END, "EOF", [24, 27],
848
- :tNL, nil, [11, 12])
849
- end
850
-
851
- def test_heredoc_double
852
- assert_scanned("a = <<\"EOF\"\n blah blah\nEOF\n",
853
- :tIDENTIFIER, "a", [0, 1],
854
- :tEQL, "=", [2, 3],
855
- :tSTRING_BEG, "<<\"", [4, 11],
856
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
857
- :tSTRING_END, "EOF", [24, 27],
858
- :tNL, nil, [11, 12])
859
- end
860
-
861
- def test_heredoc_double_dash
862
- assert_scanned("a = <<-\"EOF\"\n blah blah\n EOF\n",
863
- :tIDENTIFIER, "a", [0, 1],
864
- :tEQL, "=", [2, 3],
865
- :tSTRING_BEG, "<<\"", [4, 12],
866
- :tSTRING_CONTENT, " blah blah\n", [13, 25],
867
- :tSTRING_END, "EOF", [25, 30],
868
- :tNL, nil, [12, 13])
869
- end
870
-
871
- def test_heredoc_double_eos
872
- refute_scanned("a = <<\"EOF\"\nblah",
873
- :tIDENTIFIER, "a", [0, 1],
874
- :tEQL, "=", [2, 3],
875
- :tSTRING_BEG, "<<\"", [4, 7])
876
- end
877
-
878
- def test_heredoc_double_eos_nl
879
- refute_scanned("a = <<\"EOF\"\nblah\n",
880
- :tIDENTIFIER, "a", [0, 1],
881
- :tEQL, "=", [2, 3],
882
- :tSTRING_BEG, "<<\"", [4, 7])
883
- end
884
-
885
- def test_heredoc_double_interp
886
- assert_scanned("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
887
- :tIDENTIFIER, "a", [0, 1],
888
- :tEQL, "=", [2, 3],
889
- :tSTRING_BEG, "<<\"", [4, 11],
890
- :tSTRING_CONTENT, "#x a ", [12, 17],
891
- :tSTRING_DVAR, nil, [17, 18],
892
- :tIVAR, "@a", [18, 20],
893
- :tSTRING_CONTENT, " b ", [20, 23],
894
- :tSTRING_DVAR, nil, [23, 24],
895
- :tGVAR, "$b", [24, 26],
896
- :tSTRING_CONTENT, " c ", [26, 29],
897
- :tSTRING_DBEG, '#{', [29, 31],
898
- :tINTEGER, 3, [31, 32],
899
- :tRCURLY, "}", [32, 33],
900
- :tSTRING_CONTENT, " \n", [33, 35],
901
- :tSTRING_END, "EOF", [35, 38],
902
- :tNL, nil, [11, 12])
903
- end
904
-
905
- def test_heredoc_empty
906
- assert_scanned("<<\"\"\n\#{x}\nblah2\n\n",
907
- :tSTRING_BEG, "<<\"", [0, 4],
908
- :tSTRING_DBEG, "\#{", [5, 7],
909
- :tIDENTIFIER, "x", [7, 8],
910
- :tRCURLY, "}", [8, 9],
911
- :tSTRING_CONTENT, "\n", [9, 10],
912
- :tSTRING_CONTENT, "blah2\n", [10, 16],
913
- :tSTRING_END, "", [16, 16],
914
- :tNL, nil, [4, 5])
915
- end
916
-
917
- def test_heredoc_none
918
- assert_scanned("a = <<EOF\nblah\nblah\nEOF",
919
- :tIDENTIFIER, "a", [0, 1],
920
- :tEQL, "=", [2, 3],
921
- :tSTRING_BEG, "<<\"", [4, 9],
922
- :tSTRING_CONTENT, "blah\n", [10, 15],
923
- :tSTRING_CONTENT, "blah\n", [15, 20],
924
- :tSTRING_END, "EOF", [20, 23],
925
- :tNL, nil, [9, 10])
926
- end
927
-
928
- def test_heredoc_none_dash
929
- assert_scanned("a = <<-EOF\nblah\nblah\n EOF",
930
- :tIDENTIFIER, "a", [0, 1],
931
- :tEQL, "=", [2, 3],
932
- :tSTRING_BEG, "<<\"", [4, 10],
933
- :tSTRING_CONTENT, "blah\n", [11, 16],
934
- :tSTRING_CONTENT, "blah\n", [16, 21],
935
- :tSTRING_END, "EOF", [21, 26],
936
- :tNL, nil, [10, 11])
937
- end
938
-
939
- def test_heredoc_single
940
- assert_scanned("a = <<'EOF'\n blah blah\nEOF\n",
941
- :tIDENTIFIER, "a", [0, 1],
942
- :tEQL, "=", [2, 3],
943
- :tSTRING_BEG, "<<'", [4, 11],
944
- :tSTRING_CONTENT, " blah blah\n", [12, 24],
945
- :tSTRING_END, "EOF", [24, 27],
946
- :tNL, nil, [11, 12])
947
- end
948
-
949
- def test_heredoc_single_bad_eos_body
950
- refute_scanned("a = <<'EOF'\nblah",
951
- :tIDENTIFIER, "a", [0, 1],
952
- :tEQL, "=", [2, 3],
953
- :tSTRING_BEG, "'", [6, 7])
954
- end
955
-
956
- def test_heredoc_single_dash
957
- assert_scanned("a = <<-'EOF'\n blah blah\n EOF\n",
958
- :tIDENTIFIER, "a", [0, 1],
959
- :tEQL, "=", [2, 3],
960
- :tSTRING_BEG, "<<'", [4, 12],
961
- :tSTRING_CONTENT, " blah blah\n", [13, 25],
962
- :tSTRING_END, "EOF", [25, 30],
963
- :tNL, nil, [12, 13])
964
- end
965
-
966
- def test_heredoc_one_character
967
- assert_scanned("a = <<E\nABCDEF\nE\n",
968
- :tIDENTIFIER, "a", [0, 1],
969
- :tEQL, "=", [2, 3],
970
- :tSTRING_BEG, "<<\"", [4, 7],
971
- :tSTRING_CONTENT, "ABCDEF\n", [8, 15],
972
- :tSTRING_END, "E", [15, 16],
973
- :tNL, nil, [7, 8])
974
- end
975
-
976
- def test_heredoc_cr
977
- assert_scanned("a = <<E\r\r\nABCDEF\r\r\nE\r\r\r\n",
978
- :tIDENTIFIER, "a", [0, 1],
979
- :tEQL, "=", [2, 3],
980
- :tSTRING_BEG, "<<\"", [4, 7],
981
- :tSTRING_CONTENT, "ABCDEF\r\n", [9, 17],
982
- :tSTRING_END, "E", [17, 20],
983
- :tNL, nil, [8, 9])
984
- end
985
-
986
- def test_heredoc_with_identifier_ending_newline__19
987
- setup_lexer 19
988
- refute_scanned "<<\"EOS\n\"\n123\nEOS\n"
989
- end
990
-
991
- def test_heredoc_with_identifier_ending_newline__24
992
- setup_lexer 24
993
-
994
- assert_scanned("a = <<\"EOS\n\"\nABCDEF\nEOS\n",
995
- :tIDENTIFIER, "a", [0, 1],
996
- :tEQL, "=", [2, 3],
997
- :tSTRING_BEG, "<<\"", [4, 12],
998
- :tSTRING_CONTENT, "ABCDEF\n", [13, 20],
999
- :tSTRING_END, "EOS", [20, 23],
1000
- :tNL, nil, [12, 13])
1001
- end
1002
-
1003
- def test_heredoc_with_identifier_containing_newline_inside__19
1004
- setup_lexer 19
1005
- refute_scanned "<<\"EOS\nEOS\"\n123\nEOS\n"
1006
- end
1007
-
1008
- def test_heredoc_with_identifier_containing_newline_inside__24
1009
- setup_lexer 24
1010
-
1011
- refute_scanned "<<\"EOS\nEOS\"\n123\nEOS\n"
1012
- end
1013
-
1014
- def test_identifier
1015
- assert_scanned("identifier",
1016
- :tIDENTIFIER, "identifier", [0, 10])
1017
- end
1018
-
1019
- def test_identifier_bang
1020
- assert_scanned("identifier!",
1021
- :tFID, "identifier!", [0, 11])
1022
-
1023
- assert_scanned("identifier!=",
1024
- :tIDENTIFIER, "identifier", [0, 10],
1025
- :tNEQ, "!=", [10, 12])
1026
- end
1027
-
1028
- def test_identifier_eh
1029
- setup_lexer 19
1030
-
1031
- assert_scanned("identifier?",
1032
- :tFID, "identifier?", [0, 11])
1033
-
1034
- assert_scanned("identifier?=",
1035
- :tIDENTIFIER, "identifier", [0, 10],
1036
- :tCHARACTER, "=", [10, 12])
1037
- end
1038
-
1039
- def test_identifier_cmp
1040
- assert_lex_fname "<=>", :tCMP, [0, 3]
1041
- end
1042
-
1043
- def test_identifier_def
1044
- assert_lex_fname "identifier", :tIDENTIFIER, [0, 10]
1045
- end
1046
-
1047
- def test_identifier_equals_arrow
1048
- assert_scanned(":blah==>",
1049
- :tSYMBOL, "blah=", [0, 6],
1050
- :tASSOC, "=>", [6, 8])
1051
- end
1052
-
1053
- def test_identifier_equals3
1054
- assert_scanned(":a===b",
1055
- :tSYMBOL, "a", [0, 2],
1056
- :tEQQ, "===", [2, 5],
1057
- :tIDENTIFIER, "b", [5, 6])
1058
- end
1059
-
1060
- def test_identifier_equals_equals_arrow
1061
- assert_scanned(":a==>b",
1062
- :tSYMBOL, "a=", [0, 3],
1063
- :tASSOC, "=>", [3, 5],
1064
- :tIDENTIFIER, "b", [5, 6])
1065
- end
1066
-
1067
- def test_identifier_equals_caret
1068
- assert_lex_fname "^", :tCARET, [0, 1]
1069
- end
1070
-
1071
- def test_identifier_equals_def
1072
- assert_lex_fname "identifier=", :tIDENTIFIER, [0, 11]
1073
- end
1074
-
1075
- def test_identifier_equals_def2
1076
- assert_lex_fname "==", :tEQ, [0, 2]
1077
- end
1078
-
1079
- def test_identifier_equals_expr
1080
- @lex.state = :expr_dot
1081
- assert_scanned("y = arg",
1082
- :tIDENTIFIER, "y", [0, 1],
1083
- :tEQL, "=", [2, 3],
1084
- :tIDENTIFIER, "arg", [4, 7])
1085
-
1086
- assert_equal :expr_arg, @lex.state
1087
- end
1088
-
1089
- def test_identifier_equals_or
1090
- assert_lex_fname "|", :tPIPE, [0, 1]
1091
- end
1092
-
1093
- def test_identifier_equals_slash
1094
- assert_lex_fname "/", :tDIVIDE, [0, 1]
1095
- end
1096
-
1097
- def test_identifier_equals_tilde
1098
- @lex.state = :expr_fname
1099
- assert_scanned("identifier=~",
1100
- :tIDENTIFIER, "identifier=", [0, 11],
1101
- :tTILDE, "~", [11, 12])
1102
- end
1103
-
1104
- def test_identifier_gt
1105
- assert_lex_fname ">", :tGT, [0, 1]
1106
- end
1107
-
1108
- def test_identifier_le
1109
- assert_lex_fname "<=", :tLEQ, [0, 2]
1110
- end
1111
-
1112
- def test_identifier_lt
1113
- assert_lex_fname "<", :tLT, [0, 1]
1114
- end
1115
-
1116
- def test_identifier_tilde
1117
- assert_lex_fname "~", :tTILDE, [0, 1]
1118
- end
1119
-
1120
- def test_identifier_defined?
1121
- assert_lex_fname "defined?", :kDEFINED, [0, 8]
1122
- end
1123
-
1124
- def test_index
1125
- assert_lex_fname "[]", :tAREF, [0, 2]
1126
- end
1127
-
1128
- def test_index_equals
1129
- assert_lex_fname "[]=", :tASET, [0, 3]
1130
- end
1131
-
1132
- def test_integer
1133
- assert_scanned "42", :tINTEGER, 42, [0, 2]
1134
- end
1135
-
1136
- def test_integer_bin
1137
- assert_scanned "0b101010", :tINTEGER, 42, [0, 8]
1138
- end
1139
-
1140
- def test_integer_bin_bad_none
1141
- refute_scanned "0b "
1142
- end
1143
-
1144
- def test_integer_bin_bad_underscores
1145
- refute_scanned "0b10__01"
1146
- end
1147
-
1148
- def test_integer_dec
1149
- assert_scanned "42", :tINTEGER, 42, [0, 2]
1150
- end
1151
-
1152
- def test_integer_dec_bad_underscores
1153
- refute_scanned "42__24"
1154
- end
1155
-
1156
- def test_integer_dec_d
1157
- assert_scanned "0d42", :tINTEGER, 42, [0, 4]
1158
- end
1159
-
1160
- def test_integer_dec_d_bad_none
1161
- refute_scanned "0d"
1162
- end
1163
-
1164
- def test_integer_dec_d_bad_underscores
1165
- refute_scanned "0d42__24"
1166
- end
1167
-
1168
- def test_question_eh_a__18
1169
- setup_lexer 18
1170
-
1171
- assert_scanned "?a", :tINTEGER, 97, [0, 2]
1172
- end
1173
-
1174
- def test_question_eh_a__19
1175
- setup_lexer 19
1176
-
1177
- assert_scanned '?a', :tCHARACTER, "a", [0, 2]
1178
- end
1179
-
1180
- def test_question_eh_escape_M_escape_C__18
1181
- setup_lexer 18
1182
-
1183
- assert_scanned '?\M-\C-a', :tINTEGER, 129, [0, 8]
1184
- end
1185
-
1186
- def test_question_eh_escape_M_escape_C__19
1187
- setup_lexer 19
1188
-
1189
- assert_scanned '?\M-\C-a', :tCHARACTER, "\M-\C-a", [0, 8]
1190
- end
1191
-
1192
- def test_question_eh_escape_u_1_digit
1193
- setup_lexer 19
1194
-
1195
- refute_scanned '?\\u1'
1196
- end
1197
-
1198
- def test_question_eh_escape_u_2_digits
1199
- setup_lexer 19
1200
-
1201
- refute_scanned '?\\u12'
1202
- end
1203
-
1204
- def test_question_eh_escape_u_3_digits
1205
- setup_lexer 19
1206
-
1207
- refute_scanned '?\\u123'
1208
- end
1209
-
1210
- def test_question_eh_escape_u_4_digits
1211
- setup_lexer 19
1212
- assert_scanned '?\\u0001', :tCHARACTER, "\u0001", [0, 7]
1213
- end
1214
-
1215
- def test_question_eh_single_unicode_point
1216
- setup_lexer 19
1217
- assert_scanned '?\\u{123}', :tCHARACTER, "\u0123", [0, 8]
1218
-
1219
- setup_lexer 19
1220
- assert_scanned '?\\u{a}', :tCHARACTER, "\n", [0, 6]
1221
- end
1222
-
1223
- def test_question_eh_multiple_unicode_points
1224
- setup_lexer 19
1225
- refute_scanned '?\\u{1 2 3}'
1226
-
1227
- setup_lexer 19
1228
- refute_scanned '?\\u{a b}'
1229
- end
1230
-
1231
- def test_question_eh_escape_u_unclosed_bracket
1232
- setup_lexer 19
1233
-
1234
- refute_scanned '?\\u{123'
1235
- end
1236
-
1237
- def test_question_eh_escape_space_around_unicode_point__19
1238
- setup_lexer 19
1239
- refute_scanned '"\\u{1 }"'
1240
-
1241
- setup_lexer 19
1242
- refute_scanned '"\\u{ 1}"'
1243
-
1244
- setup_lexer 19
1245
- refute_scanned '"\\u{ 1 }"'
1246
-
1247
- setup_lexer 19
1248
- refute_scanned '"\\u{1 2 }"'
1249
-
1250
- setup_lexer 19
1251
- refute_scanned '"\\u{ 1 2}"'
1252
-
1253
- setup_lexer 19
1254
- refute_scanned '"\\u{1 2}"'
1255
- end
1256
-
1257
- def test_question_eh_escape_space_around_unicode_point__24
1258
- setup_lexer 24
1259
- assert_scanned '"\\u{ 1}"', :tSTRING, "\u0001", [0, 8]
1260
-
1261
- setup_lexer 24
1262
- assert_scanned '"\\u{1 }"', :tSTRING, "\u0001", [0, 8]
1263
-
1264
- setup_lexer 24
1265
- assert_scanned '"\\u{ 1 }"', :tSTRING, "\u0001", [0, 9]
1266
-
1267
- setup_lexer 24
1268
- assert_scanned '"\\u{1 2 }"', :tSTRING, "\u0001\u0002", [0, 10]
1269
-
1270
- setup_lexer 24
1271
- assert_scanned '"\\u{ 1 2}"', :tSTRING, "\u0001\u0002", [0, 10]
1272
-
1273
- setup_lexer 24
1274
- assert_scanned '"\\u{1 2}"', :tSTRING, "\u0001\u0002", [0, 10]
1275
- end
1276
-
1277
- def test_integer_hex
1278
- assert_scanned "0x2a", :tINTEGER, 42, [0, 4]
1279
- end
1280
-
1281
- def test_integer_hex_bad_none
1282
- refute_scanned "0x "
1283
- end
1284
-
1285
- def test_integer_hex_bad_underscores
1286
- refute_scanned "0xab__cd"
1287
- end
1288
-
1289
- def test_integer_oct
1290
- assert_scanned "052", :tINTEGER, 42, [0, 3]
1291
- end
1292
-
1293
- def test_integer_oct_bad_range
1294
- refute_scanned "08"
1295
- end
1296
-
1297
- def test_integer_oct_bad_underscores
1298
- refute_scanned "01__23"
1299
- end
1300
-
1301
- def test_integer_oct_O
1302
- assert_scanned "0O52", :tINTEGER, 42, [0, 4]
1303
- end
1304
-
1305
- def test_integer_oct_O_bad_range
1306
- refute_scanned "0O1238"
1307
- end
1308
-
1309
- def test_integer_oct_O_bad_underscores
1310
- refute_scanned "0O1__23"
1311
- end
1312
-
1313
- def test_integer_oct_O_not_bad_none
1314
- assert_scanned "0O ", :tINTEGER, 0, [0, 2]
1315
- end
1316
-
1317
- def test_integer_oct_o
1318
- assert_scanned "0o52", :tINTEGER, 42, [0, 4]
1319
- end
1320
-
1321
- def test_integer_oct_o_bad_range
1322
- refute_scanned "0o1283"
1323
- end
1324
-
1325
- def test_integer_oct_o_bad_underscores
1326
- refute_scanned "0o1__23"
1327
- end
1328
-
1329
- def test_integer_oct_o_not_bad_none
1330
- assert_scanned "0o ", :tINTEGER, 0, [0, 2]
1331
- end
1332
-
1333
- def test_integer_trailing
1334
- assert_scanned("1.to_s",
1335
- :tINTEGER, 1, [0, 1],
1336
- :tDOT, '.', [1, 2],
1337
- :tIDENTIFIER, 'to_s', [2, 6])
1338
- end
1339
-
1340
- def test_integer_underscore
1341
- assert_scanned "4_2", :tINTEGER, 42, [0, 3]
1342
- end
1343
-
1344
- def test_integer_underscore_bad
1345
- refute_scanned "4__2"
1346
- end
1347
-
1348
- def test_integer_zero
1349
- assert_scanned "0", :tINTEGER, 0, [0, 1]
1350
- end
1351
-
1352
- def test_ivar
1353
- assert_scanned "@blah", :tIVAR, "@blah", [0, 5]
1354
- end
1355
-
1356
- def test_ivar_bad
1357
- refute_scanned "@1"
1358
- end
1359
-
1360
- def test_ivar_bad_0_length
1361
- refute_scanned "1+@\n", :tINTEGER, 1, [0, 1], :tPLUS, "+", [1, 2]
1362
- end
1363
-
1364
- def test_keyword_expr
1365
- @lex.state = :expr_endarg
1366
-
1367
- assert_scanned "if", :kIF_MOD, "if", [0, 2]
1368
-
1369
- assert_equal :expr_beg, @lex.state
1370
- end
1371
-
1372
- def test_lt
1373
- assert_scanned "<", :tLT, "<", [0, 1]
1374
- end
1375
-
1376
- def test_lt2
1377
- assert_scanned("a <\< b",
1378
- :tIDENTIFIER, "a", [0, 1],
1379
- :tLSHFT, "<\<", [2, 4],
1380
- :tIDENTIFIER, "b", [5, 6])
1381
-
1382
- end
1383
-
1384
- def test_lt2_equals
1385
- assert_scanned("a <\<= b",
1386
- :tIDENTIFIER, "a", [0, 1],
1387
- :tOP_ASGN, "<\<", [2, 5],
1388
- :tIDENTIFIER, "b", [6, 7])
1389
- end
1390
-
1391
- def test_lt_equals
1392
- assert_scanned "<=", :tLEQ, "<=", [0, 2]
1393
- end
1394
-
1395
- def test_minus
1396
- assert_scanned("1 - 2",
1397
- :tINTEGER, 1, [0, 1],
1398
- :tMINUS, "-", [2, 3],
1399
- :tINTEGER, 2, [4, 5])
1400
- end
1401
-
1402
- def test_minus_equals
1403
- @lex.state = :expr_end
1404
-
1405
- assert_scanned "-=", :tOP_ASGN, "-", [0, 2]
1406
- end
1407
-
1408
- def test_minus_method
1409
- @lex.state = :expr_fname
1410
- assert_scanned "-", :tMINUS, "-", [0, 1]
1411
- end
1412
-
1413
- def test_minus_unary_method
1414
- @lex.state = :expr_fname
1415
- assert_scanned "-@", :tUMINUS, "-@", [0, 2]
1416
- end
1417
-
1418
- def test_minus_unary_number
1419
- assert_scanned("-42",
1420
- :tUNARY_NUM, "-", [0, 1],
1421
- :tINTEGER, 42, [1, 3])
1422
- end
1423
-
1424
- def test_minus_unary_whitespace_number
1425
- assert_scanned("- 42",
1426
- :tUNARY_NUM, "-", [0, 1],
1427
- :tINTEGER, 42, [2, 4])
1428
- end
1429
-
1430
- def test_nth_ref
1431
- assert_scanned('[$1, $2, $3]',
1432
- :tLBRACK, "[", [0, 1],
1433
- :tNTH_REF, 1, [1, 3], :tCOMMA, ",", [3, 4],
1434
- :tNTH_REF, 2, [5, 7], :tCOMMA, ",", [7, 8],
1435
- :tNTH_REF, 3, [9, 11],
1436
- :tRBRACK, "]", [11, 12])
1437
- end
1438
-
1439
- def test_open_bracket
1440
- assert_scanned("(", :tLPAREN, "(", [0, 1])
1441
- end
1442
-
1443
- def test_open_bracket_cmdarg
1444
- assert_scanned("m (", :tIDENTIFIER, "m", [0, 1],
1445
- :tLPAREN_ARG, "(", [2, 3])
1446
- end
1447
-
1448
- def test_open_bracket_exprarg
1449
- assert_scanned("m(", :tIDENTIFIER, "m", [0, 1],
1450
- :tLPAREN2, "(", [1, 2])
1451
- end
1452
-
1453
- def test_open_curly_bracket
1454
- assert_scanned("{",
1455
- :tLBRACE, "{", [0, 1])
1456
- end
1457
-
1458
- def test_open_curly_bracket_arg
1459
- assert_scanned("m { 3 }",
1460
- :tIDENTIFIER, "m", [0, 1],
1461
- :tLCURLY, "{", [2, 3],
1462
- :tINTEGER, 3, [4, 5],
1463
- :tRCURLY, "}", [6, 7])
1464
- end
1465
-
1466
- def test_open_curly_bracket_block
1467
- @lex.state = :expr_endarg # seen m(3)
1468
-
1469
- assert_scanned("{ 4 }",
1470
- :tLBRACE_ARG, "{", [0, 1],
1471
- :tINTEGER, 4, [2, 3],
1472
- :tRCURLY, "}", [4, 5])
1473
- end
1474
-
1475
- def test_open_square_bracket_arg
1476
- assert_scanned("m [ 3 ]",
1477
- :tIDENTIFIER, "m", [0, 1],
1478
- :tLBRACK, "[", [2, 3],
1479
- :tINTEGER, 3, [4, 5],
1480
- :tRBRACK, "]", [6, 7])
1481
- end
1482
-
1483
- def test_open_square_bracket_ary
1484
- assert_scanned("[1, 2, 3]",
1485
- :tLBRACK, "[", [0, 1],
1486
- :tINTEGER, 1, [1, 2],
1487
- :tCOMMA, ",", [2, 3],
1488
- :tINTEGER, 2, [4, 5],
1489
- :tCOMMA, ",", [5, 6],
1490
- :tINTEGER, 3, [7, 8],
1491
- :tRBRACK, "]", [8, 9])
1492
- end
1493
-
1494
- def test_open_square_bracket_meth
1495
- assert_scanned("m[3]",
1496
- :tIDENTIFIER, "m", [0, 1],
1497
- :tLBRACK2, "[", [1, 2],
1498
- :tINTEGER, 3, [2, 3],
1499
- :tRBRACK, "]", [3, 4])
1500
- end
1501
-
1502
- def test_or
1503
- assert_scanned "|", :tPIPE, "|", [0, 1]
1504
- end
1505
-
1506
- def test_or2
1507
- assert_scanned "||", :tOROP, "||", [0, 2]
1508
- end
1509
-
1510
- def test_or2__after_27
1511
- setup_lexer(27)
1512
- assert_scanned("||",
1513
- :tPIPE, "|", [0, 1],
1514
- :tPIPE, "|", [1, 2])
1515
- end
1516
-
1517
- def test_or2_equals
1518
- assert_scanned "||=", :tOP_ASGN, "||", [0, 3]
1519
- end
1520
-
1521
- def test_or_equals
1522
- assert_scanned "|=", :tOP_ASGN, "|", [0, 2]
1523
- end
1524
-
1525
- def test_percent
1526
- assert_scanned("a % 2",
1527
- :tIDENTIFIER, "a", [0, 1],
1528
- :tPERCENT, "%", [2, 3],
1529
- :tINTEGER, 2, [4, 5])
1530
- end
1531
-
1532
- def test_percent_equals
1533
- assert_scanned("a %= 2",
1534
- :tIDENTIFIER, "a", [0, 1],
1535
- :tOP_ASGN, "%", [2, 4],
1536
- :tINTEGER, 2, [5, 6])
1537
- end
1538
-
1539
- def test_plus
1540
- assert_scanned("1 + 1",
1541
- :tINTEGER, 1, [0, 1],
1542
- :tPLUS, "+", [2, 3],
1543
- :tINTEGER, 1, [4, 5])
1544
- end
1545
-
1546
- def test_plus_equals
1547
- @lex.state = :expr_end
1548
-
1549
- assert_scanned "+=", :tOP_ASGN, "+", [0, 2]
1550
- end
1551
-
1552
- def test_plus_method
1553
- @lex.state = :expr_fname
1554
- assert_scanned "+", :tPLUS, "+", [0, 1]
1555
- end
1556
-
1557
- def test_plus_unary_method
1558
- @lex.state = :expr_fname
1559
- assert_scanned "+@", :tUPLUS, "+@", [0, 2]
1560
- end
1561
-
1562
- def test_plus_unary_number
1563
- assert_scanned("+42",
1564
- :tUNARY_NUM, "+", [0, 1],
1565
- :tINTEGER, 42, [1, 3])
1566
- end
1567
-
1568
- def test_plus_unary_whitespace_number
1569
- assert_scanned("+ 42",
1570
- :tUNARY_NUM, "+", [0, 1],
1571
- :tINTEGER, 42, [2, 4])
1572
- end
1573
-
1574
- def test_numbers
1575
- assert_scanned "0b10", :tINTEGER, 2, [0, 4]
1576
- assert_scanned "0B10", :tINTEGER, 2, [0, 4]
1577
-
1578
- assert_scanned "0d10", :tINTEGER, 10, [0, 4]
1579
- assert_scanned "0D10", :tINTEGER, 10, [0, 4]
1580
-
1581
- assert_scanned "0x10", :tINTEGER, 16, [0, 4]
1582
- assert_scanned "0X10", :tINTEGER, 16, [0, 4]
1583
-
1584
- assert_scanned "0o10", :tINTEGER, 8, [0, 4]
1585
- assert_scanned "0O10", :tINTEGER, 8, [0, 4]
1586
- assert_scanned "0o", :tINTEGER, 0, [0, 2]
1587
- assert_scanned "0O", :tINTEGER, 0, [0, 2]
1588
-
1589
- assert_scanned "0o", :tINTEGER, 0, [0, 2]
1590
- assert_scanned "0O", :tINTEGER, 0, [0, 2]
1591
-
1592
- assert_scanned "0777_333", :tINTEGER, 261851, [0, 8]
1593
-
1594
- assert_scanned "0", :tINTEGER, 0, [0, 1]
1595
-
1596
- refute_scanned "0x"
1597
- refute_scanned "0X"
1598
- refute_scanned "0b"
1599
- refute_scanned "0B"
1600
- refute_scanned "0d"
1601
- refute_scanned "0D"
1602
-
1603
- refute_scanned "08"
1604
- refute_scanned "09"
1605
- refute_scanned "0o8"
1606
- refute_scanned "0o9"
1607
- refute_scanned "0O8"
1608
- refute_scanned "0O9"
1609
-
1610
- refute_scanned "1_e1"
1611
- refute_scanned "1_.1"
1612
- refute_scanned "1__1"
1613
-
1614
- refute_scanned "1end"
1615
- refute_scanned "1.1end"
1616
- end
1617
-
1618
- def test_question__18
1619
- setup_lexer 18
1620
-
1621
- assert_scanned "?*", :tINTEGER, 42, [0, 2]
1622
- end
1623
-
1624
- def test_question__19
1625
- setup_lexer 19
1626
-
1627
- assert_scanned "?*", :tCHARACTER, "*", [0, 2]
1628
- end
1629
-
1630
- def test_question_bad_eos
1631
- refute_scanned "?"
1632
- end
1633
-
1634
- def test_question_bad_ws
1635
- assert_scanned "? ", :tEH, "?", [0, 1]
1636
- assert_scanned "?\n", :tEH, "?", [0, 1]
1637
- assert_scanned "?\t", :tEH, "?", [0, 1]
1638
- assert_scanned "?\v", :tEH, "?", [0, 1]
1639
- assert_scanned "?\r", :tEH, "?", [0, 1]
1640
- assert_scanned "?\f", :tEH, "?", [0, 1]
1641
- end
1642
-
1643
- def test_question_ws_backslashed__18
1644
- setup_lexer 18
1645
-
1646
- @lex.state = :expr_beg
1647
- assert_scanned "?\\ ", :tINTEGER, 32, [0, 3]
1648
- @lex.state = :expr_beg
1649
- assert_scanned "?\\n", :tINTEGER, 10, [0, 3]
1650
- @lex.state = :expr_beg
1651
- assert_scanned "?\\t", :tINTEGER, 9, [0, 3]
1652
- @lex.state = :expr_beg
1653
- assert_scanned "?\\v", :tINTEGER, 11, [0, 3]
1654
- @lex.state = :expr_beg
1655
- assert_scanned "?\\r", :tINTEGER, 13, [0, 3]
1656
- @lex.state = :expr_beg
1657
- assert_scanned "?\\f", :tINTEGER, 12, [0, 3]
1658
- end
1659
-
1660
- def test_question_ws_backslashed__19
1661
- setup_lexer 19
1662
-
1663
- @lex.state = :expr_beg
1664
- assert_scanned "?\\ ", :tCHARACTER, " ", [0, 3]
1665
- @lex.state = :expr_beg
1666
- assert_scanned "?\\n", :tCHARACTER, "\n", [0, 3]
1667
- @lex.state = :expr_beg
1668
- assert_scanned "?\\t", :tCHARACTER, "\t", [0, 3]
1669
- @lex.state = :expr_beg
1670
- assert_scanned "?\\v", :tCHARACTER, "\v", [0, 3]
1671
- @lex.state = :expr_beg
1672
- assert_scanned "?\\r", :tCHARACTER, "\r", [0, 3]
1673
- @lex.state = :expr_beg
1674
- assert_scanned "?\\f", :tCHARACTER, "\f", [0, 3]
1675
- end
1676
-
1677
- def test_rbracket
1678
- assert_scanned "]", :tRBRACK, "]", [0, 1]
1679
- end
1680
-
1681
- def test_rcurly
1682
- assert_scanned "}", :tRCURLY, "}", [0, 1]
1683
- end
1684
-
1685
- def test_regexp
1686
- assert_scanned("/regexp/",
1687
- :tREGEXP_BEG, "/", [0, 1],
1688
- :tSTRING_CONTENT, "regexp", [1, 7],
1689
- :tSTRING_END, "/", [7, 8],
1690
- :tREGEXP_OPT, "", [8, 8])
1691
- end
1692
-
1693
- def test_regexp_ambiguous
1694
- assert_scanned("method /regexp/",
1695
- :tIDENTIFIER, "method", [0, 6],
1696
- :tREGEXP_BEG, "/", [7, 8],
1697
- :tSTRING_CONTENT, "regexp", [8, 14],
1698
- :tSTRING_END, "/", [14, 15],
1699
- :tREGEXP_OPT, "", [15, 15])
1700
- end
1701
-
1702
- def test_regexp_bad
1703
- refute_scanned("/.*/xyz",
1704
- :tREGEXP_BEG, "/", [0, 1],
1705
- :tSTRING_CONTENT, ".*", [1, 3],
1706
- :tSTRING_END, "/", [3, 4])
1707
- end
1708
-
1709
- def test_regexp_escape_C
1710
- assert_scanned('/regex\\C-x/',
1711
- :tREGEXP_BEG, "/", [0, 1],
1712
- :tSTRING_CONTENT, "regex\\C-x", [1, 10],
1713
- :tSTRING_END, "/", [10, 11],
1714
- :tREGEXP_OPT, "", [11, 11])
1715
- end
1716
-
1717
- def test_regexp_escape_C_M
1718
- assert_scanned('/regex\\C-\\M-x/',
1719
- :tREGEXP_BEG, "/", [0, 1],
1720
- :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 13],
1721
- :tSTRING_END, "/", [13, 14],
1722
- :tREGEXP_OPT, "", [14, 14])
1723
- end
1724
-
1725
- def test_regexp_escape_C_M_craaaazy
1726
- assert_scanned("/regex\\C-\\\n\\M-x/",
1727
- :tREGEXP_BEG, "/", [0, 1],
1728
- :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 15],
1729
- :tSTRING_END, "/", [15, 16],
1730
- :tREGEXP_OPT, "", [16, 16])
1731
- end
1732
-
1733
- def test_regexp_escape_C_bad_dash
1734
- refute_scanned '/regex\\Cx/', :tREGEXP_BEG, "/", [0, 1]
1735
- end
1736
-
1737
- def test_regexp_escape_C_bad_dash_eos
1738
- refute_scanned '/regex\\C-/', :tREGEXP_BEG, "/", [0, 1]
1739
- end
1740
-
1741
- def test_regexp_escape_C_bad_dash_eos2
1742
- refute_scanned '/regex\\C-', :tREGEXP_BEG, "/", [0, 1]
1743
- end
1744
-
1745
- def test_regexp_escape_C_bad_eos
1746
- refute_scanned '/regex\\C/', :tREGEXP_BEG, "/", [0, 1]
1747
- end
1748
-
1749
- def test_regexp_escape_C_bad_eos2
1750
- refute_scanned '/regex\\c', :tREGEXP_BEG, "/", [0, 1]
1751
- end
1752
-
1753
- def test_regexp_escape_M
1754
- assert_scanned('/regex\\M-x/',
1755
- :tREGEXP_BEG, "/", [0, 1],
1756
- :tSTRING_CONTENT, "regex\\M-x", [1, 10],
1757
- :tSTRING_END, "/", [10, 11],
1758
- :tREGEXP_OPT, "", [11, 11])
1759
- end
1760
-
1761
- def test_regexp_escape_M_C
1762
- assert_scanned('/regex\\M-\\C-x/',
1763
- :tREGEXP_BEG, "/", [0, 1],
1764
- :tSTRING_CONTENT, "regex\\M-\\C-x", [1, 13],
1765
- :tSTRING_END, "/", [13, 14],
1766
- :tREGEXP_OPT, "", [14, 14])
1767
- end
1768
-
1769
- def test_regexp_escape_M_bad_dash
1770
- refute_scanned '/regex\\Mx/', :tREGEXP_BEG, "/", [0, 1]
1771
- end
1772
-
1773
- def test_regexp_escape_M_bad_dash_eos
1774
- refute_scanned '/regex\\M-/', :tREGEXP_BEG, "/", [0, 1]
1775
- end
1776
-
1777
- def test_regexp_escape_M_bad_dash_eos2
1778
- refute_scanned '/regex\\M-', :tREGEXP_BEG, "/", [0, 1]
1779
- end
1780
-
1781
- def test_regexp_escape_M_bad_eos
1782
- refute_scanned '/regex\\M/', :tREGEXP_BEG, "/", [0, 1]
1783
- end
1784
-
1785
- def test_regexp_escape_backslash_slash
1786
- assert_scanned('/\\//',
1787
- :tREGEXP_BEG, "/", [0, 1],
1788
- :tSTRING_CONTENT, '/', [1, 3],
1789
- :tSTRING_END, "/", [3, 4],
1790
- :tREGEXP_OPT, "", [4, 4])
1791
- end
1792
-
1793
- def test_regexp_escape_backslash_terminator
1794
- assert_scanned('%r%blah\\%blah%',
1795
- :tREGEXP_BEG, "%r%", [0, 3],
1796
- :tSTRING_CONTENT, "blah%blah", [3, 13],
1797
- :tSTRING_END, "%", [13, 14],
1798
- :tREGEXP_OPT, "", [14, 14])
1799
- end
1800
-
1801
- def test_regexp_escape_backslash_terminator_meta1
1802
- assert_scanned('%r{blah\\}blah}',
1803
- :tREGEXP_BEG, "%r{", [0, 3],
1804
- :tSTRING_CONTENT, "blah\\}blah", [3, 13],
1805
- :tSTRING_END, "}", [13, 14],
1806
- :tREGEXP_OPT, "", [14, 14])
1807
- end
1808
-
1809
- def test_regexp_escape_backslash_terminator_meta2
1810
- assert_scanned('%r/blah\\/blah/',
1811
- :tREGEXP_BEG, "%r/", [0, 3],
1812
- :tSTRING_CONTENT, "blah/blah", [3, 13],
1813
- :tSTRING_END, "/", [13, 14],
1814
- :tREGEXP_OPT, "", [14, 14])
1815
- end
1816
-
1817
- def test_regexp_escape_backslash_terminator_meta3
1818
- assert_scanned('%r/blah\\%blah/',
1819
- :tREGEXP_BEG, "%r/", [0, 3],
1820
- :tSTRING_CONTENT, "blah\\%blah", [3, 13],
1821
- :tSTRING_END, "/", [13, 14],
1822
- :tREGEXP_OPT, "", [14, 14])
1823
- end
1824
-
1825
- def test_regexp_escape_bad_eos
1826
- refute_scanned '/regex\\', :tREGEXP_BEG, "/", [0, 1]
1827
- end
1828
-
1829
- def test_regexp_escape_bs
1830
- assert_scanned('/regex\\\\regex/',
1831
- :tREGEXP_BEG, "/", [0, 1],
1832
- :tSTRING_CONTENT, "regex\\\\regex", [1, 13],
1833
- :tSTRING_END, "/", [13, 14],
1834
- :tREGEXP_OPT, "", [14, 14])
1835
- end
1836
-
1837
- def test_regexp_escape_c
1838
- assert_scanned('/regex\\cxxx/',
1839
- :tREGEXP_BEG, "/", [0, 1],
1840
- :tSTRING_CONTENT, "regex\\cxxx", [1, 11],
1841
- :tSTRING_END, "/", [11, 12],
1842
- :tREGEXP_OPT, "", [12, 12])
1843
- end
1844
-
1845
- def test_regexp_escape_c_backslash
1846
- assert_scanned('/regex\\c\\n/',
1847
- :tREGEXP_BEG, "/", [0, 1],
1848
- :tSTRING_CONTENT, "regex\\c\\n", [1, 10],
1849
- :tSTRING_END, "/", [10, 11],
1850
- :tREGEXP_OPT, "", [11, 11])
1851
- end
1852
-
1853
- def test_regexp_escape_chars
1854
- assert_scanned('/re\\tge\\nxp/',
1855
- :tREGEXP_BEG, "/", [0, 1],
1856
- :tSTRING_CONTENT, "re\\tge\\nxp", [1, 11],
1857
- :tSTRING_END, "/", [11, 12],
1858
- :tREGEXP_OPT, "", [12, 12])
1859
- end
1860
-
1861
- def test_regexp_escape_double_backslash
1862
- assert_scanned('/[\\/\\\\]$/',
1863
- :tREGEXP_BEG, "/", [0, 1],
1864
- :tSTRING_CONTENT,'[/\\\\]$', [1, 8],
1865
- :tSTRING_END, "/", [8, 9],
1866
- :tREGEXP_OPT, "", [9, 9])
1867
- end
1868
-
1869
- def test_regexp_escape_hex
1870
- assert_scanned('/regex\\x61xp/',
1871
- :tREGEXP_BEG, "/", [0, 1],
1872
- :tSTRING_CONTENT, "regex\\x61xp", [1, 12],
1873
- :tSTRING_END, "/", [12, 13],
1874
- :tREGEXP_OPT, "", [13, 13])
1875
- end
1876
-
1877
- def test_regexp_escape_hex_bad
1878
- refute_scanned '/regex\\xzxp/', :tREGEXP_BEG, "/", [0, 1]
1879
- end
1880
-
1881
- def test_regexp_escape_hex_one
1882
- assert_scanned('/^[\\xd\\xa]{2}/on',
1883
- :tREGEXP_BEG, '/', [0, 1],
1884
- :tSTRING_CONTENT, '^[\\xd\\xa]{2}', [1, 13],
1885
- :tSTRING_END, "/", [13, 14],
1886
- :tREGEXP_OPT, 'on', [14, 16])
1887
- end
1888
-
1889
- def test_regexp_escape_oct1
1890
- assert_scanned('/regex\\0xp/',
1891
- :tREGEXP_BEG, "/", [0, 1],
1892
- :tSTRING_CONTENT, "regex\\0xp", [1, 10],
1893
- :tSTRING_END, "/", [10, 11],
1894
- :tREGEXP_OPT, "", [11, 11])
1895
- end
1896
-
1897
- def test_regexp_escape_oct2
1898
- assert_scanned('/regex\\07xp/',
1899
- :tREGEXP_BEG, "/", [0, 1],
1900
- :tSTRING_CONTENT, "regex\\07xp", [1, 11],
1901
- :tSTRING_END, "/", [11, 12],
1902
- :tREGEXP_OPT, "", [12, 12])
1903
- end
1904
-
1905
- def test_regexp_escape_oct3
1906
- assert_scanned('/regex\\10142/',
1907
- :tREGEXP_BEG, "/", [0, 1],
1908
- :tSTRING_CONTENT, "regex\\10142", [1, 12],
1909
- :tSTRING_END, "/", [12, 13],
1910
- :tREGEXP_OPT, "", [13, 13])
1911
- end
1912
-
1913
- def test_regexp_escape_return
1914
- assert_scanned("/regex\\\nregex/",
1915
- :tREGEXP_BEG, "/", [0, 1],
1916
- :tSTRING_CONTENT, "regexregex", [1, 13],
1917
- :tSTRING_END, "/", [13, 14],
1918
- :tREGEXP_OPT, "", [14, 14])
1919
- end
1920
-
1921
- def test_regexp_escape_delimiter_meta
1922
- assert_scanned("%r(\\))",
1923
- :tREGEXP_BEG, "%r(", [0, 3],
1924
- :tSTRING_CONTENT, "\\)", [3, 5],
1925
- :tSTRING_END, ")", [5, 6],
1926
- :tREGEXP_OPT, "", [6, 6])
1927
- end
1928
-
1929
- def test_regexp_escape_delimiter_nonmeta
1930
- assert_scanned("%r'\\''",
1931
- :tREGEXP_BEG, "%r'", [0, 3],
1932
- :tSTRING_CONTENT, "'", [3, 5],
1933
- :tSTRING_END, "'", [5, 6],
1934
- :tREGEXP_OPT, "", [6, 6])
1935
- end
1936
-
1937
- def test_regexp_escape_other_meta
1938
- assert_scanned("/\\.\\$\\*\\+\\.\\?\\|/",
1939
- :tREGEXP_BEG, "/", [0, 1],
1940
- :tSTRING_CONTENT, "\\.\\$\\*\\+\\.\\?\\|", [1, 15],
1941
- :tSTRING_END, "/", [15, 16],
1942
- :tREGEXP_OPT, "", [16, 16])
1943
- end
1944
-
1945
- def test_regexp_nm
1946
- assert_scanned("/.*/nm",
1947
- :tREGEXP_BEG, "/", [0, 1],
1948
- :tSTRING_CONTENT, ".*", [1, 3],
1949
- :tSTRING_END, "/", [3, 4],
1950
- :tREGEXP_OPT, "nm", [4, 6])
1951
- end
1952
-
1953
- def test_rparen
1954
- assert_scanned ")", :tRPAREN, ")", [0, 1]
1955
- end
1956
-
1957
- def test_rshft
1958
- assert_scanned("a >> 2",
1959
- :tIDENTIFIER, "a", [0, 1],
1960
- :tRSHFT, ">>", [2, 4],
1961
- :tINTEGER, 2, [5, 6])
1962
- end
1963
-
1964
- def test_rshft_equals
1965
- assert_scanned("a >>= 2",
1966
- :tIDENTIFIER, "a", [0, 1],
1967
- :tOP_ASGN, ">>", [2, 5],
1968
- :tINTEGER, 2, [6, 7])
1969
- end
1970
-
1971
- def test_star
1972
- assert_scanned("a * ",
1973
- :tIDENTIFIER, "a", [0, 1],
1974
- :tSTAR2, "*", [2, 3])
1975
-
1976
- assert_equal :expr_value, @lex.state
1977
- end
1978
-
1979
- def test_star2
1980
- assert_scanned("a ** ",
1981
- :tIDENTIFIER, "a", [0, 1],
1982
- :tPOW, "**", [2, 4])
1983
-
1984
- assert_equal :expr_value, @lex.state
1985
- end
1986
-
1987
- def test_star2_equals
1988
- assert_scanned("a **= ",
1989
- :tIDENTIFIER, "a", [0, 1],
1990
- :tOP_ASGN, "**", [2, 5])
1991
-
1992
- assert_equal :expr_beg, @lex.state
1993
- end
1994
-
1995
- def test_star2_beg
1996
- assert_scanned("** ",
1997
- :tDSTAR, "**", [0, 2])
1998
-
1999
- assert_equal :expr_beg, @lex.state
2000
- end
2001
-
2002
- def test_star_arg
2003
- @lex.state = :expr_arg
2004
-
2005
- assert_scanned(" *a",
2006
- :tSTAR, "*", [1, 2],
2007
- :tIDENTIFIER, "a", [2, 3])
2008
-
2009
- assert_equal :expr_arg, @lex.state
2010
- end
2011
-
2012
- def test_star_arg_beg
2013
- @lex.state = :expr_beg
2014
-
2015
- assert_scanned("*a",
2016
- :tSTAR, "*", [0, 1],
2017
- :tIDENTIFIER, "a", [1, 2])
2018
-
2019
- assert_equal :expr_arg, @lex.state
2020
- end
2021
-
2022
- def test_star_arg_beg_fname
2023
- @lex.state = :expr_fname
2024
-
2025
- assert_scanned("*a",
2026
- :tSTAR2, "*", [0, 1],
2027
- :tIDENTIFIER, "a", [1, 2])
2028
-
2029
- assert_equal :expr_arg, @lex.state
2030
- end
2031
-
2032
- def test_star_equals
2033
- assert_scanned("a *= ",
2034
- :tIDENTIFIER, "a", [0, 1],
2035
- :tOP_ASGN, "*", [2, 4])
2036
-
2037
- assert_equal :expr_beg, @lex.state
2038
- end
2039
-
2040
- def test_string_bad_eos
2041
- refute_scanned('%',
2042
- :tSTRING_BEG, '%', [0, 1])
2043
- end
2044
-
2045
- def test_string_bad_eos_quote
2046
- refute_scanned('%{nest',
2047
- :tSTRING_BEG, '%}', [0, 2])
2048
- end
2049
-
2050
- def test_string_double
2051
- assert_scanned('"string"',
2052
- :tSTRING, "string", [0, 8])
2053
- end
2054
-
2055
- def test_string_double_escape_C
2056
- assert_scanned('"\\C-a"',
2057
- :tSTRING, "\001", [0, 6])
2058
- end
2059
-
2060
- def test_string_double_escape_C_backslash
2061
- assert_scanned('"\\C-\\\\"',
2062
- :tSTRING, "\034", [0, 7])
2063
- end
2064
-
2065
- def test_string_double_escape_C_escape
2066
- assert_scanned('"\\C-\\M-a"',
2067
- :tSTRING, "\201", [0, 9])
2068
- end
2069
-
2070
- def test_string_double_escape_C_question
2071
- assert_scanned('"\\C-?"',
2072
- :tSTRING, "\177", [0, 6])
2073
- end
2074
-
2075
- def test_string_double_escape_M
2076
- assert_scanned('"\\M-a"',
2077
- :tSTRING, "\341", [0, 6])
2078
- end
2079
-
2080
- def test_string_double_escape_M_backslash
2081
- assert_scanned('"\\M-\\\\"',
2082
- :tSTRING, "\334", [0, 7])
2083
- end
2084
-
2085
- def test_string_double_escape_M_escape
2086
- assert_scanned('"\\M-\\C-a"',
2087
- :tSTRING, "\201", [0, 9])
2088
- end
2089
-
2090
- def test_string_double_escape_bs1
2091
- assert_scanned('"a\\a\\a"',
2092
- :tSTRING, "a\a\a", [0, 7])
2093
- end
2094
-
2095
- def test_string_double_escape_bs2
2096
- assert_scanned('"a\\\\a"',
2097
- :tSTRING, "a\\a", [0, 6])
2098
- end
2099
-
2100
- def test_string_double_escape_c
2101
- assert_scanned('"\\ca"',
2102
- :tSTRING, "\001", [0, 5])
2103
- end
2104
-
2105
- def test_string_double_escape_c_escape
2106
- assert_scanned('"\\c\\M-a"',
2107
- :tSTRING, "\201", [0, 8])
2108
- end
2109
-
2110
- def test_string_double_escape_c_question
2111
- assert_scanned('"\\c?"',
2112
- :tSTRING, "\177", [0, 5])
2113
- end
2114
-
2115
- def test_string_double_escape_chars
2116
- assert_scanned('"s\\tri\\ng"',
2117
- :tSTRING, "s\tri\ng", [0, 10])
2118
- end
2119
-
2120
- def test_string_double_escape_hex
2121
- assert_scanned('"n = \\x61\\x62\\x63"',
2122
- :tSTRING, "n = abc", [0, 18])
2123
- end
2124
-
2125
- def test_string_double_escape_octal
2126
- assert_scanned('"n = \\101\\102\\103"',
2127
- :tSTRING, "n = ABC", [0, 18])
2128
- end
2129
-
2130
- def test_string_double_escape_octal_wrap
2131
- assert_scanned('"\\753"',
2132
- :tSTRING, "\xEB", [0, 6])
2133
- end
2134
-
2135
- def test_string_double_interp
2136
- assert_scanned("\"blah #x a \#@a b \#$b c \#{3} # \"",
2137
- :tSTRING_BEG, "\"", [0, 1],
2138
- :tSTRING_CONTENT, "blah #x a ", [1, 11],
2139
- :tSTRING_DVAR, nil, [11, 12],
2140
- :tIVAR, "@a", [12, 14],
2141
- :tSTRING_CONTENT, " b ", [14, 17],
2142
- :tSTRING_DVAR, nil, [17, 18],
2143
- :tGVAR, "$b", [18, 20],
2144
- :tSTRING_CONTENT, " c ", [20, 23],
2145
- :tSTRING_DBEG, '#{', [23, 25],
2146
- :tINTEGER, 3, [25, 26],
2147
- :tRCURLY, "}", [26, 27],
2148
- :tSTRING_CONTENT, " # ", [27, 30],
2149
- :tSTRING_END, "\"", [30, 31])
2150
- end
2151
-
2152
- def test_string_double_interp_label
2153
- assert_scanned('"#{foo:bar}"',
2154
- :tSTRING_BEG, '"', [0, 1],
2155
- :tSTRING_DBEG, '#{', [1, 3],
2156
- :tIDENTIFIER, 'foo', [3, 6],
2157
- :tSYMBOL, 'bar', [6, 10],
2158
- :tRCURLY, '}', [10, 11],
2159
- :tSTRING_END, '"', [11, 12])
2160
- end
2161
-
2162
- def test_string_double_nested_curlies
2163
- assert_scanned('%{nest{one{two}one}nest}',
2164
- :tSTRING_BEG, '%{', [0, 2],
2165
- :tSTRING_CONTENT, "nest{one{two}one}nest", [2, 23],
2166
- :tSTRING_END, '}', [23, 24])
2167
- end
2168
-
2169
- def test_string_double_no_interp
2170
- assert_scanned("\"# blah\"", # pound first
2171
- :tSTRING, "# blah", [0, 8])
2172
-
2173
- assert_scanned("\"blah # blah\"", # pound not first
2174
- :tSTRING, "blah # blah", [0, 13])
2175
- end
2176
-
2177
- def test_string_escape_x_single
2178
- assert_scanned('"\\x0"',
2179
- :tSTRING, "\000", [0, 5])
2180
- end
2181
-
2182
- def test_string_pct_Q
2183
- assert_scanned("%Q[s1 s2]",
2184
- :tSTRING_BEG, '%Q[', [0, 3],
2185
- :tSTRING_CONTENT, "s1 s2", [3, 8],
2186
- :tSTRING_END, ']', [8, 9])
2187
- end
2188
-
2189
- def test_string_pct_W
2190
- assert_scanned("%W[s1 s2\ns3]",
2191
- :tWORDS_BEG, "%W[", [0, 3],
2192
- :tSTRING_CONTENT, "s1", [3, 5],
2193
- :tSPACE, nil, [5, 6],
2194
- :tSTRING_CONTENT, "s2", [6, 8],
2195
- :tSPACE, nil, [8, 9],
2196
- :tSTRING_CONTENT, "s3", [9, 11],
2197
- :tSPACE, nil, [11, 11],
2198
- :tSTRING_END, ']', [11, 12])
2199
- end
2200
-
2201
- def test_string_pct_W_bs_nl
2202
- assert_scanned("%W[s1 \\\ns2]",
2203
- :tWORDS_BEG, "%W[", [0, 3],
2204
- :tSTRING_CONTENT, "s1", [3, 5],
2205
- :tSPACE, nil, [5, 6],
2206
- :tSTRING_CONTENT, "\ns2", [6, 10],
2207
- :tSPACE, nil, [10, 10],
2208
- :tSTRING_END, ']', [10, 11])
2209
- end
2210
-
2211
- def test_string_pct_W_interp
2212
- assert_scanned('%W[#{1}#{2} #@a]',
2213
- :tWORDS_BEG, '%W[', [0, 3],
2214
- :tSTRING_DBEG, '#{', [3, 5],
2215
- :tINTEGER, 1, [5, 6],
2216
- :tRCURLY, '}', [6, 7],
2217
- :tSTRING_DBEG, '#{', [7, 9],
2218
- :tINTEGER, 2, [9, 10],
2219
- :tRCURLY, '}', [10, 11],
2220
- :tSPACE, nil, [11, 12],
2221
- :tSTRING_DVAR, nil, [12, 13],
2222
- :tIVAR, '@a', [13, 15],
2223
- :tSPACE, nil, [15, 15],
2224
- :tSTRING_END, ']', [15, 16])
2225
- end
2226
-
2227
- def test_string_pct_I
2228
- assert_scanned("%I(s1 s2)",
2229
- :tSYMBOLS_BEG, "%I(", [0, 3],
2230
- :tSTRING_CONTENT, "s1", [3, 5],
2231
- :tSPACE, nil, [5, 6],
2232
- :tSTRING_CONTENT, "s2", [6, 8],
2233
- :tSPACE, nil, [8, 8],
2234
- :tSTRING_END, ')', [8, 9])
2235
- end
2236
-
2237
- def test_string_pct_angle
2238
- assert_scanned("%<blah>",
2239
- :tSTRING_BEG, '%<', [0, 2],
2240
- :tSTRING_CONTENT, "blah", [2, 6],
2241
- :tSTRING_END, '>', [6, 7])
2242
- end
2243
-
2244
- def test_string_pct_pct
2245
- assert_scanned("%%blah%",
2246
- :tSTRING_BEG, '%%', [0, 2],
2247
- :tSTRING_CONTENT, "blah", [2, 6],
2248
- :tSTRING_END, '%', [6, 7])
2249
- end
2250
-
2251
- def test_string_pct_w
2252
- assert_scanned("%w[s1 s2 ]",
2253
- :tQWORDS_BEG, "%w[", [0, 3],
2254
- :tSTRING_CONTENT, "s1", [3, 5],
2255
- :tSPACE, nil, [5, 6],
2256
- :tSTRING_CONTENT, "s2", [6, 8],
2257
- :tSPACE, nil, [8, 9],
2258
- :tSTRING_END, "]", [9, 10])
2259
- end
2260
-
2261
- def test_string_pct_w_incomplete
2262
- refute_scanned("%w[s1 ",
2263
- :tQWORDS_BEG, "%w[", [0, 3],
2264
- :tSTRING_CONTENT, "s1", [3, 5],
2265
- :tSPACE, nil, [5, 6])
2266
- end
2267
-
2268
- def test_string_pct_w_bs_nl
2269
- assert_scanned("%w[s1 \\\ns2]",
2270
- :tQWORDS_BEG, "%w[", [0, 3],
2271
- :tSTRING_CONTENT, "s1", [3, 5],
2272
- :tSPACE, nil, [5, 6],
2273
- :tSTRING_CONTENT, "\ns2", [6, 10],
2274
- :tSPACE, nil, [10, 10],
2275
- :tSTRING_END, ']', [10, 11])
2276
- end
2277
-
2278
- def test_string_pct_w_bs_sp
2279
- assert_scanned("%w[s\\ 1 s\\ 2]",
2280
- :tQWORDS_BEG, "%w[", [0, 3],
2281
- :tSTRING_CONTENT, "s 1", [3, 7],
2282
- :tSPACE, nil, [7, 8],
2283
- :tSTRING_CONTENT, "s 2", [8, 12],
2284
- :tSPACE, nil, [12, 12],
2285
- :tSTRING_END, ']', [12, 13])
2286
- end
2287
-
2288
- def test_string_pct_w_tab
2289
- assert_scanned("%w[abc\tdef]",
2290
- :tQWORDS_BEG, "%w[", [0, 3],
2291
- :tSTRING_CONTENT, "abc", [3, 6],
2292
- :tSPACE, nil, [6, 7],
2293
- :tSTRING_CONTENT, "def", [7, 10],
2294
- :tSPACE, nil, [10, 10],
2295
- :tSTRING_END, ']', [10, 11])
2296
- end
2297
-
2298
- def test_string_pct_i
2299
- assert_scanned("%i(s1 s2)",
2300
- :tQSYMBOLS_BEG, "%i(", [0, 3],
2301
- :tSTRING_CONTENT, "s1", [3, 5],
2302
- :tSPACE, nil, [5, 6],
2303
- :tSTRING_CONTENT, "s2", [6, 8],
2304
- :tSPACE, nil, [8, 8],
2305
- :tSTRING_END, ')', [8, 9])
2306
- end
2307
-
2308
- def test_string_pct_backslash
2309
- assert_scanned("%\\a\\",
2310
- :tSTRING_BEG, "%\\", [0, 2],
2311
- :tSTRING_CONTENT, "a", [2, 3],
2312
- :tSTRING_END, "\\", [3, 4])
2313
- end
2314
-
2315
- def test_string_pct_w_backslash
2316
- assert_scanned("%w\\s1 s2 \\",
2317
- :tQWORDS_BEG, "%w\\", [0, 3],
2318
- :tSTRING_CONTENT, "s1", [3, 5],
2319
- :tSPACE, nil, [5, 6],
2320
- :tSTRING_CONTENT, "s2", [6, 8],
2321
- :tSPACE, nil, [8, 9],
2322
- :tSTRING_END, "\\", [9, 10])
2323
- end
2324
-
2325
- def test_string_pct_w_backslash_nl
2326
- assert_scanned("%w\\s1 s2 \\\n",
2327
- :tQWORDS_BEG, "%w\\", [0, 3],
2328
- :tSTRING_CONTENT, "s1", [3, 5],
2329
- :tSPACE, nil, [5, 6],
2330
- :tSTRING_CONTENT, "s2", [6, 8],
2331
- :tSPACE, nil, [8, 9],
2332
- :tSTRING_END, "\\", [9, 10],
2333
- :tNL, nil, [10, 11])
2334
- end
2335
-
2336
- def test_string_pct_w_backslash_interp_nl
2337
- assert_scanned("%W\\blah #x a \#@a b \#$b c \#{3} # \\",
2338
- :tWORDS_BEG, "%W\\", [0, 3],
2339
- :tSTRING_CONTENT, "blah", [3, 7],
2340
- :tSPACE, nil, [7, 8],
2341
- :tSTRING_CONTENT, "#x", [8, 10],
2342
- :tSPACE, nil, [10, 11],
2343
- :tSTRING_CONTENT, "a", [11, 12],
2344
- :tSPACE, nil, [12, 13],
2345
- :tSTRING_DVAR, nil, [13, 14],
2346
- :tIVAR, "@a", [14, 16],
2347
- :tSPACE, nil, [16, 17],
2348
- :tSTRING_CONTENT, "b", [17, 18],
2349
- :tSPACE, nil, [18, 19],
2350
- :tSTRING_DVAR, nil, [19, 20],
2351
- :tGVAR, "$b", [20, 22],
2352
- :tSPACE, nil, [22, 23],
2353
- :tSTRING_CONTENT, "c", [23, 24],
2354
- :tSPACE, nil, [24, 25],
2355
- :tSTRING_DBEG, '#{', [25, 27],
2356
- :tINTEGER, 3, [27, 28],
2357
- :tRCURLY, "}", [28, 29],
2358
- :tSPACE, nil, [29, 30],
2359
- :tSTRING_CONTENT, "#", [30, 31],
2360
- :tSPACE, nil, [31, 32],
2361
- :tSTRING_END, "\\", [32, 33])
2362
- end
2363
-
2364
- def test_string_pct_backslash_with_bad_escape
2365
- # No escapes are allowed in a backslash-delimited string
2366
- refute_scanned("%\\a\\n\\",
2367
- :tSTRING_BEG, "%\\", [0, 2],
2368
- :tSTRING_CONTENT, "a", [2, 3],
2369
- :tSTRING_END, "\\", [3, 4],
2370
- :tIDENTIFIER, "n", [4, 5])
2371
- end
2372
-
2373
- def test_string_pct_intertwined_with_heredoc
2374
- assert_scanned("<<-foo + %\\a\nbar\nfoo\nb\\",
2375
- :tSTRING_BEG, "<<\"", [0, 6],
2376
- :tSTRING_CONTENT, "bar\n", [13, 17],
2377
- :tSTRING_END, "foo", [17, 20],
2378
- :tPLUS, "+", [7, 8],
2379
- :tSTRING_BEG, "%\\", [9, 11],
2380
- :tSTRING_CONTENT, "a\n", [11, 13],
2381
- :tSTRING_CONTENT, "b", [21, 22],
2382
- :tSTRING_END, "\\", [22, 23])
2383
- end
2384
-
2385
- def test_string_pct_q_backslash
2386
- assert_scanned("%q\\a\\",
2387
- :tSTRING_BEG, "%q\\", [0, 3],
2388
- :tSTRING_CONTENT, "a", [3, 4],
2389
- :tSTRING_END, "\\", [4, 5])
2390
- end
2391
-
2392
- def test_string_pct_Q_backslash
2393
- assert_scanned("%Q\\a\\",
2394
- :tSTRING_BEG, "%Q\\", [0, 3],
2395
- :tSTRING_CONTENT, "a", [3, 4],
2396
- :tSTRING_END, "\\", [4, 5])
2397
- end
2398
-
2399
- def test_string_single
2400
- assert_scanned("'string'",
2401
- :tSTRING, "string", [0, 8])
2402
- end
2403
-
2404
- def test_string_single_escape_chars
2405
- assert_scanned("'s\\tri\\ng'",
2406
- :tSTRING, "s\\tri\\ng", [0, 10])
2407
- end
2408
-
2409
- def test_string_single_nl
2410
- assert_scanned("'blah\\\nblah'",
2411
- :tSTRING_BEG, "'", [0, 1],
2412
- :tSTRING_CONTENT, "blah\\\n", [1, 7],
2413
- :tSTRING_CONTENT, "blah", [7, 11],
2414
- :tSTRING_END, "'", [11, 12])
2415
- end
2416
-
2417
- def test_symbol
2418
- assert_scanned(":symbol",
2419
- :tSYMBOL, "symbol", [0, 7])
2420
- end
2421
-
2422
- def test_symbol_double
2423
- assert_scanned(":\"symbol\"",
2424
- :tSYMBEG, ":\"", [0, 2],
2425
- :tSTRING_CONTENT, "symbol", [2, 8],
2426
- :tSTRING_END, "\"", [8, 9])
2427
- end
2428
-
2429
- def test_symbol_single
2430
- assert_scanned(":'symbol'",
2431
- :tSYMBEG, ":'", [0, 2],
2432
- :tSTRING_CONTENT, "symbol", [2, 8],
2433
- :tSTRING_END, "'", [8, 9])
2434
- end
2435
-
2436
- def test_ternary
2437
- assert_scanned("a ? b : c",
2438
- :tIDENTIFIER, "a", [0, 1],
2439
- :tEH, "?", [2, 3],
2440
- :tIDENTIFIER, "b", [4, 5],
2441
- :tCOLON, ":", [6, 7],
2442
- :tIDENTIFIER, "c", [8, 9])
2443
-
2444
- assert_scanned("a ?b : c",
2445
- :tIDENTIFIER, "a", [0, 1],
2446
- :tINTEGER, 98, [2, 4],
2447
- :tCOLON, ":", [5, 6],
2448
- :tIDENTIFIER, "c", [7, 8])
2449
-
2450
- assert_scanned("a ?bb : c", # GAH! MATZ!!!
2451
- :tIDENTIFIER, "a", [0, 1],
2452
- :tEH, "?", [2, 3],
2453
- :tIDENTIFIER, "bb", [3, 5],
2454
- :tCOLON, ":", [6, 7],
2455
- :tIDENTIFIER, "c", [8, 9])
2456
-
2457
- assert_scanned("42 ?", # 42 forces expr_end
2458
- :tINTEGER, 42, [0, 2],
2459
- :tEH, "?", [3, 4])
2460
- end
2461
-
2462
- def test_tilde
2463
- assert_scanned "~", :tTILDE, "~", [0, 1]
2464
- end
2465
-
2466
- def test_tilde_unary
2467
- @lex.state = :expr_fname
2468
- assert_scanned "~@", :tTILDE, "~@", [0, 2]
2469
- end
2470
-
2471
- def test_uminus
2472
- assert_scanned("-blah",
2473
- :tUMINUS, "-", [0, 1],
2474
- :tIDENTIFIER, "blah", [1, 5])
2475
- end
2476
-
2477
- def test_underscore
2478
- assert_scanned("_var", :tIDENTIFIER, "_var", [0, 4])
2479
- end
2480
-
2481
- def test_underscore_end
2482
- assert_scanned("__END__\n")
2483
- assert_scanned("__END__")
2484
- assert_scanned("__END__ foo",
2485
- :tIDENTIFIER, '__END__', [0, 7],
2486
- :tIDENTIFIER, 'foo', [8, 11])
2487
- assert_scanned("__END__\rfoo",
2488
- :tIDENTIFIER, '__END__', [0, 7],
2489
- :tIDENTIFIER, 'foo', [8, 11])
2490
- end
2491
-
2492
- def test_uplus
2493
- assert_scanned("+blah",
2494
- :tUPLUS, "+", [0, 1],
2495
- :tIDENTIFIER, "blah", [1, 5])
2496
- end
2497
-
2498
- def test_if_unless_mod
2499
- assert_scanned("return if true unless false",
2500
- :kRETURN, "return", [0, 6],
2501
- :kIF_MOD, "if", [7, 9],
2502
- :kTRUE, "true", [10, 14],
2503
- :kUNLESS_MOD, "unless", [15, 21],
2504
- :kFALSE, "false", [22, 27])
2505
- end
2506
-
2507
- def test_if_stmt
2508
- assert_scanned("if true\n return end",
2509
- :kIF, "if", [0, 2],
2510
- :kTRUE, "true", [3, 7],
2511
- :tNL, nil, [7, 8],
2512
- :kRETURN, "return", [9, 15],
2513
- :kEND, "end", [16, 19])
2514
- end
2515
-
2516
- def test_sclass_label
2517
- setup_lexer 20
2518
- assert_scanned("class << a:b",
2519
- :kCLASS, 'class', [0, 5],
2520
- :tLSHFT, '<<', [6, 8],
2521
- :tIDENTIFIER, 'a', [9, 10],
2522
- :tSYMBOL, 'b', [10, 12])
2523
- end
2524
-
2525
- def test_fname_pct_s__22
2526
- setup_lexer 22
2527
- @lex.state = :expr_fname
2528
- assert_scanned("%s(a)",
2529
- :tPERCENT, '%', [0, 1],
2530
- :tIDENTIFIER, 's', [1, 2],
2531
- :tLPAREN2, '(', [2, 3],
2532
- :tIDENTIFIER, 'a', [3, 4],
2533
- :tRPAREN, ')', [4, 5])
2534
- end
2535
-
2536
- def test_fname_pct_s__23
2537
- setup_lexer 23
2538
- @lex.state = :expr_fname
2539
- assert_scanned("%s(a)",
2540
- :tSYMBEG, '%s(', [0, 3],
2541
- :tSTRING_CONTENT, 'a', [3, 4],
2542
- :tSTRING_END, ')', [4, 5])
2543
- end
2544
-
2545
- def test_static_env
2546
- env = Parser::StaticEnvironment.new
2547
- env.declare "a"
2548
-
2549
- @lex.static_env = env
2550
- assert_scanned("a [42]",
2551
- :tIDENTIFIER, "a", [0, 1],
2552
- :tLBRACK2, "[", [2, 3],
2553
- :tINTEGER, 42, [3, 5],
2554
- :tRBRACK, "]", [5, 6])
2555
- end
2556
-
2557
- def test_int_suffix
2558
- [18, 19, 20].each do |version|
2559
- setup_lexer version
2560
-
2561
- assert_scanned("42r",
2562
- :tINTEGER, 42, [0, 2],
2563
- :tIDENTIFIER, 'r', [2, 3])
2564
-
2565
- assert_scanned("42if",
2566
- :tINTEGER, 42, [0, 2],
2567
- :kIF_MOD, 'if', [2, 4])
2568
- end
2569
-
2570
- setup_lexer 21
2571
-
2572
- assert_scanned("42r", :tRATIONAL, Rational(42), [0, 3])
2573
- assert_scanned("42i", :tIMAGINARY, Complex(0, 42), [0, 3])
2574
- assert_scanned("42ri", :tIMAGINARY, Complex(0, Rational(42)), [0, 4])
2575
- end
2576
-
2577
- def test_float_suffix
2578
- [18, 19, 20].each do |version|
2579
- setup_lexer version
2580
-
2581
- assert_scanned("42.1r",
2582
- :tFLOAT, 42.1, [0, 4],
2583
- :tIDENTIFIER, 'r', [4, 5])
2584
-
2585
- assert_scanned("42.1if",
2586
- :tFLOAT, 42.1, [0, 4],
2587
- :kIF_MOD, 'if', [4, 6])
2588
-
2589
- assert_scanned("1e1r",
2590
- :tFLOAT, 1e1, [0, 3],
2591
- :tIDENTIFIER, 'r', [3, 4])
2592
- end
2593
-
2594
- begin
2595
- # Feature-check.
2596
- Rational("10")
2597
-
2598
- setup_lexer 21
2599
-
2600
- assert_scanned("42.1r", :tRATIONAL, Rational(421, 10), [0, 5])
2601
- assert_scanned("42.1i", :tIMAGINARY, Complex(0, 42.1), [0, 5])
2602
- assert_scanned("42.1ri", :tIMAGINARY, Complex(0, Rational(421, 10)), [0, 6])
2603
- assert_scanned("42.1ir",
2604
- :tIMAGINARY, Complex(0, 42.1), [0, 5],
2605
- :tIDENTIFIER, 'r', [5, 6])
2606
-
2607
- assert_scanned("1e1i", :tIMAGINARY, Complex(0, 1e1), [0, 4])
2608
- assert_scanned("1e1r",
2609
- :tFLOAT, 1e1, [0, 3],
2610
- :tIDENTIFIER, 'r', [3, 4])
2611
- assert_scanned("1e1ri",
2612
- :tFLOAT, 1e1, [0, 3],
2613
- :tIDENTIFIER, 'ri', [3, 5])
2614
- assert_scanned("1e1ir",
2615
- :tIMAGINARY, Complex(0, 1e1), [0, 4],
2616
- :tIDENTIFIER, 'r', [4, 5])
2617
- rescue NoMethodError
2618
- # Ruby not modern enough
2619
- end
2620
- end
2621
-
2622
- def test_eof
2623
- assert_scanned("self",
2624
- :kSELF, "self", [0, 4])
2625
- assert_equal([false, ["$eof", Parser::Source::Range.new(@lex.source_buffer, 4, 4)]],
2626
- @lex.advance)
2627
- end
2628
-
2629
- #
2630
- # Test for 'fluent interface'
2631
- #
2632
-
2633
- def test_fluent_dot
2634
- assert_scanned("x\n.y",
2635
- :tIDENTIFIER, 'x', [0, 1],
2636
- :tDOT, '.', [2, 3],
2637
- :tIDENTIFIER, 'y', [3, 4])
2638
-
2639
- assert_scanned("x\n .y",
2640
- :tIDENTIFIER, 'x', [0, 1],
2641
- :tDOT, '.', [4, 5],
2642
- :tIDENTIFIER, 'y', [5, 6])
2643
-
2644
- assert_scanned("x # comment\n .y",
2645
- :tIDENTIFIER, 'x', [0, 1],
2646
- :tDOT, '.', [14, 15],
2647
- :tIDENTIFIER, 'y', [15, 16])
2648
- end
2649
-
2650
- def test_fluent_and_dot
2651
- assert_scanned("x\n&.y",
2652
- :tIDENTIFIER, 'x', [0, 1],
2653
- :tANDDOT, '&.', [2, 4],
2654
- :tIDENTIFIER, 'y', [4, 5])
2655
- end
2656
-
2657
- #
2658
- # Tests for whitespace.
2659
- #
2660
-
2661
- def test_whitespace_fname
2662
- @lex.state = :expr_fname
2663
- assert_scanned('class',
2664
- :kCLASS, 'class', [0, 5])
2665
-
2666
- @lex.state = :expr_fname
2667
- assert_scanned(' class',
2668
- :kCLASS, 'class', [1, 6])
2669
-
2670
- @lex.state = :expr_fname
2671
- assert_scanned("\nclass",
2672
- :kCLASS, 'class', [1, 6])
2673
-
2674
- @lex.state = :expr_fname
2675
- assert_scanned("\\\nclass",
2676
- :kCLASS, 'class', [2, 7])
2677
-
2678
- @lex.state = :expr_fname
2679
- assert_scanned("#foo\nclass",
2680
- :kCLASS, 'class', [5, 10])
2681
- end
2682
-
2683
- def test_whitespace_endfn
2684
- setup_lexer(21)
2685
-
2686
- @lex.state = :expr_endfn
2687
- assert_scanned('foo:',
2688
- :tLABEL, 'foo', [0, 4])
2689
-
2690
- @lex.state = :expr_endfn
2691
- assert_scanned(' foo:',
2692
- :tLABEL, 'foo', [1, 5])
2693
-
2694
- @lex.state = :expr_endfn
2695
- assert_scanned("\nfoo:",
2696
- :tNL, nil, [0, 1],
2697
- :tIDENTIFIER, 'foo', [1, 4],
2698
- :tCOLON, ':', [4, 5])
2699
-
2700
- @lex.state = :expr_endfn
2701
- assert_scanned("\nfoo: ",
2702
- :tNL, nil, [0, 1],
2703
- :tIDENTIFIER, 'foo', [1, 4],
2704
- :tCOLON, ':', [4, 5])
2705
-
2706
- @lex.state = :expr_endfn
2707
- assert_scanned("\\\nfoo:",
2708
- :tLABEL, 'foo', [2, 6])
2709
-
2710
- @lex.state = :expr_endfn
2711
- assert_scanned("#foo\nfoo:",
2712
- :tNL, nil, [4, 5],
2713
- :tIDENTIFIER, 'foo', [5, 8],
2714
- :tCOLON, ':', [8, 9])
2715
-
2716
- @lex.state = :expr_endfn
2717
- assert_scanned("#foo\nfoo: ",
2718
- :tNL, nil, [4, 5],
2719
- :tIDENTIFIER, 'foo', [5, 8],
2720
- :tCOLON, ':', [8, 9])
2721
- end
2722
-
2723
- def test_whitespace_dot
2724
- @lex.state = :expr_dot
2725
- assert_scanned('class',
2726
- :tIDENTIFIER, 'class', [0, 5])
2727
-
2728
- @lex.state = :expr_dot
2729
- assert_scanned(' class',
2730
- :tIDENTIFIER, 'class', [1, 6])
2731
-
2732
- @lex.state = :expr_dot
2733
- assert_scanned("\nclass",
2734
- :tIDENTIFIER, 'class', [1, 6])
2735
-
2736
- @lex.state = :expr_dot
2737
- assert_scanned("\\\nclass",
2738
- :tIDENTIFIER, 'class', [2, 7])
2739
-
2740
- @lex.state = :expr_dot
2741
- assert_scanned("#foo\nclass",
2742
- :tIDENTIFIER, 'class', [5, 10])
2743
- end
2744
-
2745
- def test_whitespace_arg
2746
- @lex.state = :expr_arg
2747
- assert_scanned('+',
2748
- :tPLUS, '+', [0, 1])
2749
-
2750
- @lex.state = :expr_arg
2751
- assert_scanned(' +',
2752
- :tUPLUS, '+', [1, 2])
2753
-
2754
- @lex.state = :expr_arg
2755
- assert_scanned("\n+",
2756
- :tNL, nil, [0, 1],
2757
- :tUPLUS, '+', [1, 2])
2758
-
2759
- @lex.state = :expr_arg
2760
- assert_scanned("\\\n+",
2761
- :tUPLUS, '+', [2, 3])
2762
-
2763
- @lex.state = :expr_arg
2764
- assert_scanned("\\\n +",
2765
- :tUPLUS, '+', [3, 4])
2766
-
2767
- @lex.state = :expr_arg
2768
- assert_scanned("#foo\n+",
2769
- :tNL, nil, [4, 5],
2770
- :tUPLUS, '+', [5, 6])
2771
- end
2772
-
2773
- def test_whitespace_endarg
2774
- @lex.state = :expr_endarg
2775
- assert_scanned('{',
2776
- :tLBRACE_ARG, '{', [0, 1])
2777
-
2778
- @lex.state = :expr_endarg
2779
- assert_scanned(' {',
2780
- :tLBRACE_ARG, '{', [1, 2])
2781
-
2782
- @lex.state = :expr_endarg
2783
- assert_scanned("\n{",
2784
- :tNL, nil, [0, 1],
2785
- :tLBRACE, '{', [1, 2])
2786
-
2787
- @lex.state = :expr_endarg
2788
- assert_scanned("\\\n{",
2789
- :tLBRACE_ARG, '{', [2, 3])
2790
-
2791
- @lex.state = :expr_endarg
2792
- assert_scanned("#foo\n{",
2793
- :tNL, nil, [4, 5],
2794
- :tLBRACE, '{', [5, 6])
2795
- end
2796
-
2797
- def test_whitespace_mid
2798
- @lex.state = :expr_mid
2799
- assert_scanned('+',
2800
- :tUPLUS, '+', [0, 1])
2801
-
2802
- @lex.state = :expr_mid
2803
- assert_scanned(' +',
2804
- :tUPLUS, '+', [1, 2])
2805
-
2806
- @lex.state = :expr_mid
2807
- assert_scanned("\n+",
2808
- :tNL, nil, [0, 1],
2809
- :tUPLUS, '+', [1, 2])
2810
-
2811
- @lex.state = :expr_mid
2812
- assert_scanned("\\\n+",
2813
- :tUPLUS, '+', [2, 3])
2814
-
2815
- @lex.state = :expr_mid
2816
- assert_scanned("#foo\n+",
2817
- :tNL, nil, [4, 5],
2818
- :tUPLUS, '+', [5, 6])
2819
- end
2820
-
2821
- def test_whitespace_beg
2822
- @lex.state = :expr_beg
2823
- assert_scanned('+',
2824
- :tUPLUS, '+', [0, 1])
2825
-
2826
- @lex.state = :expr_beg
2827
- assert_scanned(' +',
2828
- :tUPLUS, '+', [1, 2])
2829
-
2830
- @lex.state = :expr_beg
2831
- assert_scanned("\n+",
2832
- :tUPLUS, '+', [1, 2])
2833
-
2834
- @lex.state = :expr_beg
2835
- assert_scanned("\\\n+",
2836
- :tUPLUS, '+', [2, 3])
2837
-
2838
- @lex.state = :expr_beg
2839
- assert_scanned("#foo\n+",
2840
- :tUPLUS, '+', [5, 6])
2841
- end
2842
-
2843
- def test_whitespace_value
2844
- setup_lexer(20)
2845
-
2846
- @lex.state = :expr_value
2847
- assert_scanned('a:b',
2848
- :tIDENTIFIER, 'a', [0, 1],
2849
- :tSYMBOL, 'b', [1, 3])
2850
-
2851
- @lex.state = :expr_value
2852
- assert_scanned(' a:b',
2853
- :tIDENTIFIER, 'a', [1, 2],
2854
- :tSYMBOL, 'b', [2, 4])
2855
-
2856
- @lex.state = :expr_value
2857
- assert_scanned("\na:b",
2858
- :tIDENTIFIER, 'a', [1, 2],
2859
- :tSYMBOL, 'b', [2, 4])
2860
-
2861
- @lex.state = :expr_value
2862
- assert_scanned("\\\na:b",
2863
- :tIDENTIFIER, 'a', [2, 3],
2864
- :tSYMBOL, 'b', [3, 5])
2865
-
2866
- @lex.state = :expr_value
2867
- assert_scanned("#foo\na:b",
2868
- :tIDENTIFIER, 'a', [5, 6],
2869
- :tSYMBOL, 'b', [6, 8])
2870
- end
2871
-
2872
- def test_whitespace_end
2873
- @lex.state = :expr_end
2874
- assert_scanned('+ 1',
2875
- :tPLUS, '+', [0, 1],
2876
- :tINTEGER, 1, [2, 3])
2877
-
2878
- @lex.state = :expr_end
2879
- assert_scanned(' + 1',
2880
- :tPLUS, '+', [1, 2],
2881
- :tINTEGER, 1, [3, 4])
2882
-
2883
- @lex.state = :expr_end
2884
- assert_scanned("\n+ 1",
2885
- :tNL, nil, [0, 1],
2886
- :tUNARY_NUM, '+', [1, 2],
2887
- :tINTEGER, 1, [3, 4])
2888
-
2889
- @lex.state = :expr_end
2890
- assert_scanned("\\\n+ 1",
2891
- :tPLUS, '+', [2, 3],
2892
- :tINTEGER, 1, [4, 5])
2893
-
2894
- @lex.state = :expr_end
2895
- assert_scanned("#foo\n+ 1",
2896
- :tNL, nil, [4, 5],
2897
- :tUNARY_NUM, '+', [5, 6],
2898
- :tINTEGER, 1, [7, 8])
2899
- end
2900
-
2901
- def test_whitespace_cr
2902
- setup_lexer(20)
2903
- assert_scanned("<<E\nfoo\nE\rO",
2904
- :tSTRING_BEG, '<<"', [0, 3],
2905
- :tSTRING_CONTENT, "foo\n", [4, 8],
2906
- :tSTRING_END, 'E', [8, 11],
2907
- :tNL, nil, [3, 4])
2908
-
2909
- setup_lexer(21)
2910
- refute_scanned("<<E\nfoo\nE\rO",
2911
- :tSTRING_BEG, '<<"', [0, 3],
2912
- :tSTRING_CONTENT, "foo\n", [4, 8])
2913
- end
2914
-
2915
- #
2916
- # Handling of encoding-related issues.
2917
- #
2918
-
2919
- def test_transcoded_source_is_converted_back_to_original_encoding
2920
- setup_lexer(19)
2921
- @lex.force_utf32 = true
2922
- @lex.tokens = []
2923
- assert_scanned(utf('"a" + "b"'),
2924
- :tSTRING, "a", [0, 3],
2925
- :tPLUS, "+", [4, 5],
2926
- :tSTRING, "b", [6, 9])
2927
-
2928
- @lex.tokens.each do |_type, (str, _range)|
2929
- assert_equal Encoding::UTF_8, str.encoding
2930
- end
2931
- end
2932
-
2933
- #
2934
- # Tests for bugs.
2935
- #
2936
- # These tests should be moved from nursery and properly
2937
- # categorized when it's clear how to do that.
2938
- #
2939
-
2940
- def test_bug_sclass_joined
2941
- assert_scanned("class<<self",
2942
- :kCLASS, "class", [0, 5],
2943
- :tLSHFT, "<<", [5, 7],
2944
- :kSELF, "self", [7, 11])
2945
- end
2946
-
2947
- def test_bug_const_expr_end
2948
- assert_scanned("Option",
2949
- :tCONSTANT, 'Option', [0, 6])
2950
-
2951
- assert_equal :expr_cmdarg, @lex.state
2952
- end
2953
-
2954
- def test_bug_expr_beg_div
2955
- @lex.state = :expr_beg
2956
- assert_scanned("/=/",
2957
- :tREGEXP_BEG, "/", [0, 1],
2958
- :tSTRING_CONTENT, "=", [1, 2],
2959
- :tSTRING_END, "/", [2, 3],
2960
- :tREGEXP_OPT, "", [3, 3])
2961
-
2962
- @lex.state = :expr_beg
2963
- assert_scanned("/ = /",
2964
- :tREGEXP_BEG, "/", [0, 1],
2965
- :tSTRING_CONTENT, " = ", [1, 4],
2966
- :tSTRING_END, "/", [4, 5],
2967
- :tREGEXP_OPT, "", [5, 5])
2968
- end
2969
-
2970
- def test_bug_expr_beg_percent
2971
- @lex.state = :expr_beg
2972
- assert_scanned("%=foo=",
2973
- :tSTRING_BEG, "%=", [0, 2],
2974
- :tSTRING_CONTENT, 'foo', [2, 5],
2975
- :tSTRING_END, "=", [5, 6])
2976
-
2977
- @lex.state = :expr_beg
2978
- assert_scanned("% = ",
2979
- :tSTRING_BEG, "% ", [0, 2],
2980
- :tSTRING_CONTENT, '=', [2, 3],
2981
- :tSTRING_END, ' ', [3, 4])
2982
- end
2983
-
2984
- def test_bug_expr_beg_document
2985
- @lex.state = :expr_beg
2986
- assert_scanned(" \n=begin\n=end\nend",
2987
- :kEND, "end", [14, 17])
2988
-
2989
- end
2990
-
2991
- def test_bug_expr_beg_number
2992
- @lex.state = :expr_beg
2993
- assert_scanned("86400_000_000",
2994
- :tINTEGER, 86400_000_000, [0, 13])
2995
- end
2996
-
2997
- def test_bug_expr_beg_backspace_nl
2998
- @lex.state = :expr_beg
2999
- assert_scanned("\n/foo/",
3000
- :tREGEXP_BEG, "/", [1, 2],
3001
- :tSTRING_CONTENT, "foo", [2, 5],
3002
- :tSTRING_END, "/", [5, 6],
3003
- :tREGEXP_OPT, "", [6, 6])
3004
- end
3005
-
3006
- def test_bug_expr_beg_heredoc
3007
- assert_scanned("<<EOL % [\nfoo\nEOL\n]",
3008
- :tSTRING_BEG, '<<"', [0, 5],
3009
- :tSTRING_CONTENT, "foo\n", [10, 14],
3010
- :tSTRING_END, 'EOL', [14, 17],
3011
- :tPERCENT, '%', [6, 7],
3012
- :tLBRACK, '[', [8, 9],
3013
- :tRBRACK, ']', [18, 19])
3014
- end
3015
-
3016
- def test_bug_expr_beg_fid
3017
- assert_scanned("Rainbows!",
3018
- :tFID, 'Rainbows!', [0, 9])
3019
- end
3020
-
3021
- def test_bug_expr_beg_rescue_assoc
3022
- assert_scanned("rescue=>",
3023
- :kRESCUE, 'rescue', [0, 6],
3024
- :tASSOC, '=>', [6, 8])
3025
- end
3026
-
3027
- def test_bug_expr_arg_percent
3028
- @lex.state = :expr_arg
3029
- assert_scanned("%[",
3030
- :tPERCENT, "%", [0, 1],
3031
- :tLBRACK, "[", [1, 2])
3032
-
3033
- @lex.state = :expr_arg
3034
- assert_scanned("%=1",
3035
- :tOP_ASGN, "%", [0, 2],
3036
- :tINTEGER, 1, [2, 3])
3037
-
3038
- @lex.state = :expr_arg
3039
- assert_scanned(" %[1]",
3040
- :tSTRING_BEG, "%[", [1, 3],
3041
- :tSTRING_CONTENT, '1', [3, 4],
3042
- :tSTRING_END, ']', [4, 5])
3043
-
3044
- @lex.state = :expr_arg
3045
- assert_scanned(" %=1=",
3046
- :tOP_ASGN, "%", [1, 3],
3047
- :tINTEGER, 1, [3, 4],
3048
- :tEQL, "=", [4, 5])
3049
-
3050
- @lex.state = :expr_arg
3051
- assert_scanned(" %\n",
3052
- :tPERCENT, '%', [1, 2])
3053
- end
3054
-
3055
- def test_bug_expr_arg_lt_lt
3056
- @lex.state = :expr_arg
3057
- assert_scanned("<<EOS\nEOS",
3058
- :tLSHFT, "<<", [0, 2],
3059
- :tCONSTANT, "EOS", [2, 5],
3060
- :tNL, nil, [5, 6],
3061
- :tCONSTANT, "EOS", [6, 9])
3062
-
3063
- @lex.state = :expr_arg
3064
- assert_scanned(" <<EOS\nEOS",
3065
- :tSTRING_BEG, "<<\"", [1, 6],
3066
- :tSTRING_END, "EOS", [7, 10],
3067
- :tNL, nil, [6, 7])
3068
- end
3069
-
3070
- def test_bug_expr_arg_slash
3071
- @lex.state = :expr_arg
3072
- assert_scanned("/1",
3073
- :tDIVIDE, "/", [0, 1],
3074
- :tINTEGER, 1, [1, 2])
3075
-
3076
- @lex.state = :expr_arg
3077
- assert_scanned("/ 1",
3078
- :tDIVIDE, "/", [0, 1],
3079
- :tINTEGER, 1, [2, 3])
3080
-
3081
- @lex.state = :expr_arg
3082
- assert_scanned(" /1/",
3083
- :tREGEXP_BEG, "/", [1, 2],
3084
- :tSTRING_CONTENT, "1", [2, 3],
3085
- :tSTRING_END, "/", [3, 4],
3086
- :tREGEXP_OPT, "", [4, 4])
3087
-
3088
- @lex.state = :expr_arg
3089
- assert_scanned(" / 1",
3090
- :tDIVIDE, "/", [1, 2],
3091
- :tINTEGER, 1, [3, 4])
3092
- end
3093
-
3094
- def test_bug_expr_arg_label
3095
- setup_lexer 19
3096
-
3097
- @lex.state = :expr_arg
3098
- assert_scanned(" unless:",
3099
- :tLABEL, 'unless', [1, 8])
3100
-
3101
- @lex.state = :expr_arg
3102
- assert_scanned(" unless: ",
3103
- :tLABEL, 'unless', [1, 8])
3104
- end
3105
-
3106
- def test_bug_heredoc_continuation
3107
- @lex.state = :expr_arg
3108
- assert_scanned(" <<EOS\nEOS\nend",
3109
- :tSTRING_BEG, "<<\"", [1, 6],
3110
- :tSTRING_END, "EOS", [7, 10],
3111
- :tNL, nil, [6, 7],
3112
- :kEND, "end", [11, 14])
3113
- end
3114
-
3115
- def test_bug_heredoc_cr_lf
3116
- assert_scanned("<<FIN\r\nfoo\r\nFIN\r\n",
3117
- :tSTRING_BEG, "<<\"", [0, 5],
3118
- :tSTRING_CONTENT, "foo\n", [6, 10],
3119
- :tSTRING_END, "FIN", [10, 13],
3120
- :tNL, nil, [5, 6])
3121
- end
3122
-
3123
- def test_bug_eh_symbol_no_newline
3124
- assert_scanned("?\"\nfoo",
3125
- :tINTEGER, 34, [0, 2],
3126
- :tNL, nil, [2, 3],
3127
- :tIDENTIFIER, "foo", [3, 6])
3128
- end
3129
-
3130
- def test_bug_expr_arg_newline
3131
- @lex.state = :expr_arg
3132
- assert_scanned("\nfoo",
3133
- :tNL, nil, [0, 1],
3134
- :tIDENTIFIER, "foo", [1, 4])
3135
-
3136
- @lex.state = :expr_arg
3137
- assert_scanned(" \nfoo",
3138
- :tNL, nil, [1, 2],
3139
- :tIDENTIFIER, "foo", [2, 5])
3140
-
3141
- @lex.state = :expr_arg
3142
- assert_scanned("#foo\nfoo",
3143
- :tNL, nil, [4, 5],
3144
- :tIDENTIFIER, "foo", [5, 8])
3145
- end
3146
-
3147
- def test_bug_expr_arg_comment_newline
3148
- @lex.state = :expr_arg
3149
- assert_scanned(" #\nfoo",
3150
- :tNL, nil, [2, 3],
3151
- :tIDENTIFIER, 'foo', [3, 6])
3152
- end
3153
-
3154
- def test_bug_expr_arg_eh_crlf
3155
- @lex.state = :expr_arg
3156
- assert_scanned(" ?\r\n",
3157
- :tEH, '?', [1, 2])
3158
- end
3159
-
3160
- def test_bug_heredoc_backspace_nl
3161
- assert_scanned(" <<'XXX'\nf \\\nXXX\n",
3162
- :tSTRING_BEG, "<<'", [1, 8],
3163
- :tSTRING_CONTENT, "f \\\n", [9, 13],
3164
- :tSTRING_END, "XXX", [13, 16],
3165
- :tNL, nil, [8, 9])
3166
- end
3167
-
3168
- def test_bug_heredoc_lshft
3169
- assert_scanned("<<RULES << CLEANINGS\nRULES",
3170
- :tSTRING_BEG, '<<"', [0, 7],
3171
- :tSTRING_END, 'RULES', [21, 26],
3172
- :tLSHFT, '<<', [8, 10],
3173
- :tCONSTANT, 'CLEANINGS', [11, 20])
3174
- end
3175
-
3176
- def test_bug_sclass_comment_lshft_label
3177
- assert_scanned("class # foo\n<< a:b;end",
3178
- :kCLASS, 'class', [0, 5],
3179
- :tLSHFT, '<<', [12, 14],
3180
- :tIDENTIFIER, 'a', [15, 16],
3181
- :tSYMBOL, 'b', [16, 18],
3182
- :tSEMI, ';', [18, 19],
3183
- :kEND, 'end', [19, 22])
3184
- end
3185
-
3186
- def test_bug_expr_dot_comment
3187
- assert_scanned("foo. #bar\nbaz",
3188
- :tIDENTIFIER, 'foo', [0, 3],
3189
- :tDOT, '.', [3, 4],
3190
- :tIDENTIFIER, 'baz', [10, 13])
3191
- end
3192
-
3193
- def test_bug_expr_dot_fid
3194
- assert_scanned("foo.S?",
3195
- :tIDENTIFIER, 'foo', [0, 3],
3196
- :tDOT, '.', [3, 4],
3197
- :tFID, 'S?', [4, 6])
3198
- end
3199
-
3200
- def test_bug_expr_dot_id_eq
3201
- assert_scanned("foo.x= 1",
3202
- :tIDENTIFIER, 'foo', [0, 3],
3203
- :tDOT, '.', [3, 4],
3204
- :tIDENTIFIER, 'x', [4, 5],
3205
- :tEQL, '=', [5, 6],
3206
- :tINTEGER, 1, [7, 8])
3207
- end
3208
-
3209
- def test_bug_expr_dot_fid_mod
3210
- assert_scanned("foo.x!if 1",
3211
- :tIDENTIFIER, 'foo', [0, 3],
3212
- :tDOT, '.', [3, 4],
3213
- :tFID, 'x!', [4, 6],
3214
- :kIF_MOD, 'if', [6, 8],
3215
- :tINTEGER, 1, [9, 10])
3216
- end
3217
-
3218
- def test_bug_expr_mid_comment
3219
- assert_scanned("rescue #bar\nprint",
3220
- :kRESCUE, 'rescue', [0, 6],
3221
- :tNL, nil, [11, 12],
3222
- :tIDENTIFIER, 'print', [12, 17])
3223
- end
3224
-
3225
- def test_bug_expr_mid_bareword
3226
- assert_scanned("begin; rescue rescue1",
3227
- :kBEGIN, 'begin', [0, 5],
3228
- :tSEMI, ';', [5, 6],
3229
- :kRESCUE, 'rescue', [7, 13],
3230
- :tIDENTIFIER, 'rescue1', [14, 21])
3231
- end
3232
-
3233
- def test_bug_expr_value_document
3234
- assert_scanned("1;\n=begin\n=end",
3235
- :tINTEGER, 1, [0, 1],
3236
- :tSEMI, ';', [1, 2])
3237
- end
3238
-
3239
- def test_bug_expr_end_colon
3240
- assert_scanned("'foo':'bar'",
3241
- :tSTRING, 'foo', [0, 5],
3242
- :tCOLON, ':', [5, 6],
3243
- :tSTRING, 'bar', [6, 11])
3244
- end
3245
-
3246
- def test_bug_expr_value_rescue_colon2
3247
- @lex.state = :expr_value
3248
- assert_scanned("rescue::Exception",
3249
- :kRESCUE, 'rescue', [0, 6],
3250
- :tCOLON3, '::', [6, 8],
3251
- :tCONSTANT, 'Exception', [8, 17])
3252
- end
3253
-
3254
- def test_bug_expr_endarg_braces
3255
- assert_scanned("let [] {",
3256
- :tIDENTIFIER, 'let', [0, 3],
3257
- :tLBRACK, '[', [4, 5],
3258
- :tRBRACK, ']', [5, 6],
3259
- :tLBRACE_ARG, '{', [7, 8])
3260
- end
3261
-
3262
- def test_bug_line_begin_label
3263
- setup_lexer(19)
3264
- assert_scanned("foo:bar",
3265
- :tIDENTIFIER, 'foo', [0, 3],
3266
- :tSYMBOL, 'bar', [3, 7])
3267
- end
3268
-
3269
- def test_bug_interp_expr_value
3270
- assert_scanned('"#{f:a}"',
3271
- :tSTRING_BEG, '"', [0, 1],
3272
- :tSTRING_DBEG, '#{', [1, 3],
3273
- :tIDENTIFIER, 'f', [3, 4],
3274
- :tSYMBOL, 'a', [4, 6],
3275
- :tRCURLY, '}', [6, 7],
3276
- :tSTRING_END, '"', [7, 8])
3277
- end
3278
-
3279
- def test_bug_const_e
3280
- assert_scanned('E10',
3281
- :tCONSTANT, 'E10', [0, 3])
3282
- assert_scanned('E4U',
3283
- :tCONSTANT, 'E4U', [0, 3])
3284
- end
3285
-
3286
- def test_bug_symbol_newline
3287
- assert_scanned(":foo\n",
3288
- :tSYMBOL, 'foo', [0, 4],
3289
- :tNL, nil, [4, 5])
3290
-
3291
- assert_scanned(":foo=\n",
3292
- :tSYMBOL, 'foo=', [0, 5],
3293
- :tNL, nil, [5, 6])
3294
- end
3295
-
3296
- def test_bug_interleaved_heredoc
3297
- assert_scanned(%Q{<<w; "\nfoo\nw\n"},
3298
- :tSTRING_BEG, '<<"', [0, 3],
3299
- :tSTRING_CONTENT, "foo\n", [7, 11],
3300
- :tSTRING_END, 'w', [11, 12],
3301
- :tSEMI, ';', [3, 4],
3302
- :tSTRING_BEG, '"', [5, 6],
3303
- :tSTRING_CONTENT, "\n", [6, 7],
3304
- :tSTRING_END, '"', [13, 14])
3305
-
3306
- @lex.state = :expr_beg
3307
- assert_scanned(%Q{<<w; %w[\nfoo\nw\n1]},
3308
- :tSTRING_BEG, '<<"', [0, 3],
3309
- :tSTRING_CONTENT, "foo\n", [9, 13],
3310
- :tSTRING_END, 'w', [13, 14],
3311
- :tSEMI, ';', [3, 4],
3312
- :tQWORDS_BEG, '%w[', [5, 8],
3313
- :tSTRING_CONTENT, "1", [15, 16],
3314
- :tSPACE, nil, [16, 16],
3315
- :tSTRING_END, ']', [16, 17])
3316
-
3317
- @lex.state = :expr_beg
3318
- assert_scanned(%Q{<<w; "\#{\nfoo\nw\n}"},
3319
- :tSTRING_BEG, '<<"', [0, 3],
3320
- :tSTRING_CONTENT, "foo\n", [9, 13],
3321
- :tSTRING_END, 'w', [13, 14],
3322
- :tSEMI, ';', [3, 4],
3323
- :tSTRING_BEG, '"', [5, 6],
3324
- :tSTRING_DBEG, '#{', [6, 8],
3325
- :tRCURLY, '}', [15, 16],
3326
- :tSTRING_END, '"', [16, 17])
3327
- end
3328
-
3329
- def test_bug_fid_char
3330
- setup_lexer(19)
3331
- assert_scanned('eof??a',
3332
- :tFID, 'eof?', [0, 4],
3333
- :tCHARACTER, 'a', [4, 6])
3334
- end
3335
-
3336
- def test_bug_nonlabel_context__18
3337
- env = Parser::StaticEnvironment.new
3338
- env.declare "a"
3339
-
3340
- @lex.static_env = env
3341
- assert_scanned("1+a:a",
3342
- :tINTEGER, 1, [0, 1],
3343
- :tPLUS, '+', [1, 2],
3344
- :tIDENTIFIER, 'a', [2, 3],
3345
- :tCOLON, ':', [3, 4],
3346
- :tIDENTIFIER, 'a', [4, 5])
3347
- end
3348
-
3349
- def test_bug_string_percent_newline
3350
- assert_scanned(%Q{%\nfoo\n},
3351
- :tSTRING_BEG, "%\n", [0, 2],
3352
- :tSTRING_CONTENT, 'foo', [2, 5],
3353
- :tSTRING_END, "\n", [5, 6])
3354
- end
3355
-
3356
- def test_bug_string_percent_zero
3357
- assert_scanned(%Q{%\0foo\0},
3358
- :tSTRING_BEG, "%\0", [0, 2],
3359
- :tSTRING_CONTENT, 'foo', [2, 5],
3360
- :tSTRING_END, "\0", [5, 6])
3361
- end
3362
-
3363
- def test_bug_string_utf_escape_composition
3364
- assert_scanned(%q{"\xE2\x80\x99"},
3365
- :tSTRING, "\xE2\x80\x99", [0, 14])
3366
-
3367
- assert_scanned(utf(%q{"\xE2\x80\x99"}),
3368
- :tSTRING, utf('’'), [0, 14])
3369
- assert_scanned(utf(%q{"\342\200\231"}),
3370
- :tSTRING, utf('’'), [0, 14])
3371
- assert_scanned(utf(%q{"\M-b\C-\M-@\C-\M-Y"}),
3372
- :tSTRING, utf('’'), [0, 20])
3373
- end
3374
-
3375
- def test_bug_string_utf_escape_noop
3376
- assert_scanned(utf(%q{"\あ"}),
3377
- :tSTRING, utf("あ"), [0, 4])
3378
- end
3379
-
3380
- def test_bug_string_non_utf
3381
- assert_scanned(%Q{"caf\xE9"},
3382
- :tSTRING, "caf\xE9", [0, 6])
3383
- assert_scanned(%Q{"caf\xC3\xA9"},
3384
- :tSTRING, "caf\xC3\xA9", [0, 7])
3385
-
3386
- assert_scanned(utf(%q{"café"}),
3387
- :tSTRING, utf("café"), [0, 6])
3388
- end
3389
-
3390
- def test_bug_semi__END__
3391
- assert_scanned(%Q{foo;\n__END__},
3392
- :tIDENTIFIER, 'foo', [0, 3],
3393
- :tSEMI, ';', [3, 4])
3394
- end
3395
-
3396
- def test_bug_eql_end
3397
- assert_scanned(%Q{=begin\n#=end\n=end})
3398
- end
3399
-
3400
- def test_bug_hidden_eof
3401
- @lex.state = :expr_beg
3402
- assert_scanned(%Q{"foo\0\x1a\x04bar"},
3403
- :tSTRING_BEG, '"', [0, 1],
3404
- :tSTRING_CONTENT, "foo\0", [1, 5],
3405
- :tSTRING_CONTENT, "\x1a", [5, 6],
3406
- :tSTRING_CONTENT, "\x04", [6, 7],
3407
- :tSTRING_CONTENT, "bar", [7, 10],
3408
- :tSTRING_END, '"', [10, 11])
3409
-
3410
- @lex.state = :expr_beg
3411
- assert_scanned(%Q{'foo\0\x1a\x04bar'},
3412
- :tSTRING_BEG, "'", [0, 1],
3413
- :tSTRING_CONTENT, "foo\0", [1, 5],
3414
- :tSTRING_CONTENT, "\x1a", [5, 6],
3415
- :tSTRING_CONTENT, "\x04", [6, 7],
3416
- :tSTRING_CONTENT, "bar", [7, 10],
3417
- :tSTRING_END, "'", [10, 11])
3418
-
3419
- @lex.state = :expr_beg
3420
- assert_scanned(%Q{%w[foo\0\x1a\x04bar]},
3421
- :tQWORDS_BEG, '%w[', [0, 3],
3422
- :tSTRING_CONTENT, "foo\0", [3, 7],
3423
- :tSTRING_CONTENT, "\x1a", [7, 8],
3424
- :tSTRING_CONTENT, "\x04", [8, 9],
3425
- :tSTRING_CONTENT, "bar", [9, 12],
3426
- :tSPACE, nil, [12, 12],
3427
- :tSTRING_END, ']', [12, 13])
3428
-
3429
- @lex.state = :expr_beg
3430
- assert_scanned(%Q{%W[foo\0\x1a\x04bar]},
3431
- :tWORDS_BEG, '%W[', [0, 3],
3432
- :tSTRING_CONTENT, "foo\0", [3, 7],
3433
- :tSTRING_CONTENT, "\x1a", [7, 8],
3434
- :tSTRING_CONTENT, "\x04", [8, 9],
3435
- :tSTRING_CONTENT, "bar", [9, 12],
3436
- :tSPACE, nil, [12, 12],
3437
- :tSTRING_END, ']', [12, 13])
3438
-
3439
- @lex.state = :expr_beg
3440
- assert_scanned(%Q{# foo\0\nbar},
3441
- :tIDENTIFIER, 'bar', [7, 10])
3442
-
3443
- @lex.state = :line_begin
3444
- assert_scanned(%Q{=begin\n\0\n=end\nbar},
3445
- :tIDENTIFIER, 'bar', [14, 17])
3446
- end
3447
-
3448
- def test_bug_num_adj_kw
3449
- assert_scanned('1if',
3450
- :tINTEGER, 1, [0, 1],
3451
- :kIF_MOD, 'if', [1, 3])
3452
-
3453
- assert_scanned('1.0if',
3454
- :tFLOAT, 1.0, [0, 3],
3455
- :kIF_MOD, 'if', [3, 5])
3456
- end
3457
-
3458
- def test_bug_unicode_in_literal
3459
- setup_lexer(19)
3460
- assert_scanned('"\u00a4"',
3461
- :tSTRING, "\u00a4", [0, 8])
3462
- end
3463
-
3464
- def test_bug_utf32le_leak
3465
- setup_lexer(19)
3466
- @lex.force_utf32 = true
3467
- assert_scanned('"F0"',
3468
- :tSTRING, "F0", [0, 4])
3469
- end
3470
-
3471
- def test_bug_ragel_stack
3472
- assert_scanned("\"\#{$2 ? $2 : 1}\"",
3473
- :tSTRING_BEG, "\"", [0, 1],
3474
- :tSTRING_DBEG, "\#{", [1, 3],
3475
- :tNTH_REF, 2, [3, 5],
3476
- :tEH, "?", [6, 7],
3477
- :tNTH_REF, 2, [8, 10],
3478
- :tCOLON, ":", [11, 12],
3479
- :tINTEGER, 1, [13, 14],
3480
- :tRCURLY, "}", [14, 15],
3481
- :tSTRING_END, "\"", [15, 16])
3482
- end
3483
-
3484
- def test_bug_423
3485
- @lex.state = :expr_beg
3486
- assert_scanned(':&&',
3487
- :tSYMBEG, ':', [0, 1],
3488
- :tANDOP, '&&', [1, 3])
3489
-
3490
- @lex.state = :expr_beg
3491
- assert_scanned(':||',
3492
- :tSYMBEG, ':', [0, 1],
3493
- :tOROP, '||', [1, 3])
3494
- end
3495
-
3496
- def test_bug_418
3497
- setup_lexer 19
3498
-
3499
- assert_scanned("{\n=begin\nx: 1,\n=end\ny: 2}",
3500
- :tLBRACE, '{', [0, 1],
3501
- :tLABEL, 'y', [20, 22],
3502
- :tINTEGER, 2, [23, 24],
3503
- :tRCURLY, '}', [24, 25])
3504
- end
3505
-
3506
- def test_bug_407
3507
- setup_lexer(21)
3508
-
3509
- assert_scanned('123if cond',
3510
- :tINTEGER, 123, [0, 3],
3511
- :kIF_MOD, 'if', [3, 5],
3512
- :tIDENTIFIER, 'cond', [6, 10])
3513
-
3514
- assert_scanned('1.23if cond',
3515
- :tFLOAT, 1.23, [0, 4],
3516
- :kIF_MOD, 'if', [4, 6],
3517
- :tIDENTIFIER, 'cond', [7, 11])
3518
-
3519
- assert_scanned('123rescue cond',
3520
- :tINTEGER, 123, [0, 3],
3521
- :kRESCUE_MOD, 'rescue', [3, 9],
3522
- :tIDENTIFIER, 'cond', [10, 14])
3523
-
3524
- assert_scanned('1.23rescue cond',
3525
- :tFLOAT, 1.23, [0, 4],
3526
- :kRESCUE_MOD, 'rescue', [4, 10],
3527
- :tIDENTIFIER, 'cond', [11, 15])
3528
- end
3529
-
3530
- def test_parser_bug_486
3531
- setup_lexer(19)
3532
- assert_scanned(':!@',
3533
- :tSYMBOL, '!', [0, 3])
3534
-
3535
- setup_lexer(19)
3536
- assert_scanned(':~@',
3537
- :tSYMBOL, '~', [0, 3])
3538
- end
3539
-
3540
- def test_slash_only_in_heredocs
3541
- setup_lexer(23)
3542
- refute_scanned(%Q{<<~E\n\\\nE})
3543
-
3544
- setup_lexer(23)
3545
- refute_scanned(%Q{<<-E\n\\\nE})
3546
- end
3547
-
3548
- def test_escapes_in_squiggly_heredoc
3549
- setup_lexer(23)
3550
-
3551
- assert_scanned(%Q{<<~E\n\a\b\e\f\r\t\\\v\nE},
3552
- :tSTRING_BEG, '<<"', [0, 4],
3553
- :tSTRING_CONTENT, "\a\b\e\f\r\t\v\n", [5, 14],
3554
- :tSTRING_END, 'E', [14, 15],
3555
- :tNL, nil, [4, 5])
3556
-
3557
- setup_lexer(23)
3558
-
3559
- assert_scanned(%Q{<<-E\n\a\b\e\f\r\t\\\v\nE},
3560
- :tSTRING_BEG, '<<"', [0, 4],
3561
- :tSTRING_CONTENT, "\a\b\e\f\r\t\v\n", [5, 14],
3562
- :tSTRING_END, 'E', [14, 15],
3563
- :tNL, nil, [4, 5])
3564
- end
3565
-
3566
- def test_ambiguous_integer_re
3567
- assert_scanned('1re',
3568
- :tINTEGER, 1, [0, 1],
3569
- :tIDENTIFIER, 're', [1, 3])
3570
- end
3571
-
3572
- def lex_numbered_parameter(input)
3573
- @lex.max_numparam_stack.push
3574
-
3575
- @lex.context = Parser::Context.new
3576
- @lex.context.push(:block)
3577
-
3578
- source_buffer = Parser::Source::Buffer.new('(assert_lex_numbered_parameter)')
3579
- source_buffer.source = input
3580
-
3581
- @lex.source_buffer = source_buffer
3582
-
3583
- @lex.advance
3584
- end
3585
-
3586
- def assert_scanned_numbered_parameter(input)
3587
- lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
3588
-
3589
- assert_equal(lex_token, :tNUMPARAM)
3590
- assert_equal(lex_value, input.tr('@', ''))
3591
- assert_equal(lex_range.begin_pos, 0)
3592
- assert_equal(lex_range.end_pos, input.length)
3593
- end
3594
-
3595
- def refute_scanned_numbered_parameter(input, message = nil)
3596
- err = assert_raises Parser::SyntaxError do
3597
- lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
3598
- end
3599
-
3600
- if message
3601
- assert_equal(err.message, Parser::MESSAGES[message])
3602
-
3603
- assert_equal(err.diagnostic.location.begin_pos, 0)
3604
- assert_equal(err.diagnostic.location.end_pos, input.length)
3605
- end
3606
- end
3607
-
3608
- end