regexp_parser 1.4.0 → 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -1
  3. data/Gemfile +1 -1
  4. data/README.md +9 -13
  5. data/lib/regexp_parser/expression.rb +33 -21
  6. data/lib/regexp_parser/expression/classes/backref.rb +18 -10
  7. data/lib/regexp_parser/expression/classes/conditional.rb +4 -0
  8. data/lib/regexp_parser/expression/classes/group.rb +4 -2
  9. data/lib/regexp_parser/expression/classes/keep.rb +1 -3
  10. data/lib/regexp_parser/expression/methods/match_length.rb +172 -0
  11. data/lib/regexp_parser/expression/quantifier.rb +2 -2
  12. data/lib/regexp_parser/expression/sequence.rb +0 -4
  13. data/lib/regexp_parser/expression/subexpression.rb +3 -5
  14. data/lib/regexp_parser/lexer.rb +31 -24
  15. data/lib/regexp_parser/parser.rb +25 -3
  16. data/lib/regexp_parser/syntax/tokens.rb +2 -10
  17. data/lib/regexp_parser/version.rb +1 -1
  18. data/regexp_parser.gemspec +2 -2
  19. data/spec/expression/base_spec.rb +80 -0
  20. data/spec/expression/clone_spec.rb +120 -0
  21. data/spec/expression/conditional_spec.rb +89 -0
  22. data/spec/expression/free_space_spec.rb +27 -0
  23. data/spec/expression/methods/match_length_spec.rb +141 -0
  24. data/spec/expression/methods/strfregexp_spec.rb +224 -0
  25. data/spec/expression/methods/tests_spec.rb +97 -0
  26. data/spec/expression/methods/traverse_spec.rb +140 -0
  27. data/spec/expression/subexpression_spec.rb +50 -0
  28. data/spec/expression/to_h_spec.rb +26 -0
  29. data/spec/expression/to_s_spec.rb +100 -0
  30. data/spec/lexer/all_spec.rb +22 -0
  31. data/{test/lexer/test_conditionals.rb → spec/lexer/conditionals_spec.rb} +31 -35
  32. data/spec/lexer/escapes_spec.rb +38 -0
  33. data/spec/lexer/keep_spec.rb +22 -0
  34. data/{test/lexer/test_literals.rb → spec/lexer/literals_spec.rb} +20 -24
  35. data/{test/lexer/test_nesting.rb → spec/lexer/nesting_spec.rb} +11 -13
  36. data/spec/lexer/refcalls_spec.rb +54 -0
  37. data/spec/parser/all_spec.rb +31 -0
  38. data/spec/parser/alternation_spec.rb +88 -0
  39. data/{test/parser/test_anchors.rb → spec/parser/anchors_spec.rb} +7 -10
  40. data/spec/parser/conditionals_spec.rb +179 -0
  41. data/spec/parser/errors_spec.rb +51 -0
  42. data/spec/parser/escapes_spec.rb +132 -0
  43. data/spec/parser/free_space_spec.rb +130 -0
  44. data/spec/parser/groups_spec.rb +267 -0
  45. data/spec/parser/keep_spec.rb +19 -0
  46. data/spec/parser/posix_classes_spec.rb +27 -0
  47. data/spec/parser/properties_spec.rb +127 -0
  48. data/spec/parser/quantifiers_spec.rb +293 -0
  49. data/spec/parser/refcalls_spec.rb +237 -0
  50. data/spec/parser/set/intersections_spec.rb +127 -0
  51. data/spec/parser/set/ranges_spec.rb +111 -0
  52. data/spec/parser/sets_spec.rb +178 -0
  53. data/{test/parser/test_types.rb → spec/parser/types_spec.rb} +13 -20
  54. data/spec/scanner/all_spec.rb +18 -0
  55. data/{test/scanner/test_anchors.rb → spec/scanner/anchors_spec.rb} +8 -10
  56. data/{test/scanner/test_conditionals.rb → spec/scanner/conditionals_spec.rb} +49 -53
  57. data/spec/scanner/errors_spec.rb +90 -0
  58. data/{test/scanner/test_escapes.rb → spec/scanner/escapes_spec.rb} +8 -10
  59. data/{test/scanner/test_free_space.rb → spec/scanner/free_space_spec.rb} +48 -52
  60. data/{test/scanner/test_groups.rb → spec/scanner/groups_spec.rb} +33 -41
  61. data/spec/scanner/keep_spec.rb +33 -0
  62. data/{test/scanner/test_literals.rb → spec/scanner/literals_spec.rb} +8 -12
  63. data/{test/scanner/test_meta.rb → spec/scanner/meta_spec.rb} +8 -10
  64. data/{test/scanner/test_properties.rb → spec/scanner/properties_spec.rb} +14 -19
  65. data/{test/scanner/test_quantifiers.rb → spec/scanner/quantifiers_spec.rb} +7 -9
  66. data/{test/scanner/test_refcalls.rb → spec/scanner/refcalls_spec.rb} +9 -9
  67. data/{test/scanner/test_scripts.rb → spec/scanner/scripts_spec.rb} +8 -12
  68. data/{test/scanner/test_sets.rb → spec/scanner/sets_spec.rb} +14 -17
  69. data/spec/scanner/types_spec.rb +29 -0
  70. data/spec/scanner/unicode_blocks_spec.rb +28 -0
  71. data/spec/spec_helper.rb +14 -0
  72. data/{test → spec}/support/runner.rb +9 -8
  73. data/{test → spec}/support/warning_extractor.rb +5 -7
  74. data/spec/syntax/syntax_spec.rb +44 -0
  75. data/spec/syntax/syntax_token_map_spec.rb +23 -0
  76. data/spec/syntax/versions/1.8.6_spec.rb +38 -0
  77. data/spec/syntax/versions/1.9.1_spec.rb +23 -0
  78. data/spec/syntax/versions/1.9.3_spec.rb +22 -0
  79. data/spec/syntax/versions/2.0.0_spec.rb +28 -0
  80. data/spec/syntax/versions/2.2.0_spec.rb +22 -0
  81. data/spec/syntax/versions/aliases_spec.rb +119 -0
  82. data/spec/token/token_spec.rb +85 -0
  83. metadata +131 -140
  84. data/test/expression/test_all.rb +0 -12
  85. data/test/expression/test_base.rb +0 -90
  86. data/test/expression/test_clone.rb +0 -89
  87. data/test/expression/test_conditionals.rb +0 -113
  88. data/test/expression/test_free_space.rb +0 -35
  89. data/test/expression/test_set.rb +0 -84
  90. data/test/expression/test_strfregexp.rb +0 -230
  91. data/test/expression/test_subexpression.rb +0 -58
  92. data/test/expression/test_tests.rb +0 -99
  93. data/test/expression/test_to_h.rb +0 -59
  94. data/test/expression/test_to_s.rb +0 -104
  95. data/test/expression/test_traverse.rb +0 -161
  96. data/test/helpers.rb +0 -10
  97. data/test/lexer/test_all.rb +0 -41
  98. data/test/lexer/test_keep.rb +0 -24
  99. data/test/lexer/test_refcalls.rb +0 -56
  100. data/test/parser/set/test_intersections.rb +0 -127
  101. data/test/parser/set/test_ranges.rb +0 -111
  102. data/test/parser/test_all.rb +0 -64
  103. data/test/parser/test_alternation.rb +0 -92
  104. data/test/parser/test_conditionals.rb +0 -187
  105. data/test/parser/test_errors.rb +0 -63
  106. data/test/parser/test_escapes.rb +0 -134
  107. data/test/parser/test_free_space.rb +0 -139
  108. data/test/parser/test_groups.rb +0 -289
  109. data/test/parser/test_keep.rb +0 -21
  110. data/test/parser/test_posix_classes.rb +0 -27
  111. data/test/parser/test_properties.rb +0 -134
  112. data/test/parser/test_quantifiers.rb +0 -301
  113. data/test/parser/test_refcalls.rb +0 -186
  114. data/test/parser/test_sets.rb +0 -179
  115. data/test/scanner/test_all.rb +0 -38
  116. data/test/scanner/test_errors.rb +0 -91
  117. data/test/scanner/test_keep.rb +0 -35
  118. data/test/scanner/test_types.rb +0 -35
  119. data/test/scanner/test_unicode_blocks.rb +0 -30
  120. data/test/support/disable_autotest.rb +0 -8
  121. data/test/syntax/test_all.rb +0 -6
  122. data/test/syntax/test_syntax.rb +0 -61
  123. data/test/syntax/test_syntax_token_map.rb +0 -25
  124. data/test/syntax/versions/test_1.8.rb +0 -55
  125. data/test/syntax/versions/test_1.9.1.rb +0 -36
  126. data/test/syntax/versions/test_1.9.3.rb +0 -32
  127. data/test/syntax/versions/test_2.0.0.rb +0 -37
  128. data/test/syntax/versions/test_2.2.0.rb +0 -32
  129. data/test/syntax/versions/test_aliases.rb +0 -129
  130. data/test/syntax/versions/test_all.rb +0 -5
  131. data/test/test_all.rb +0 -5
  132. data/test/token/test_all.rb +0 -2
  133. data/test/token/test_token.rb +0 -107
@@ -1,25 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
-
3
- class TestSyntaxTokenMap < Test::Unit::TestCase
4
- def test_syntax_token_map_complete
5
- map = Regexp::Syntax::Token::Map
6
- latest_syntax = Regexp::Syntax.new 'ruby/2.9'
7
-
8
- latest_syntax.features.each do |type, tokens|
9
- tokens.each do |token|
10
- assert map[type].include?(token),
11
- "`#{type} #{token}` implemented but missing from Map"
12
- end
13
- end
14
- end
15
-
16
- def test_syntax_token_map_uniq
17
- combinations = Regexp::Syntax::Token::Map.flat_map do |type, tokens|
18
- tokens.map { |token| "#{type} #{token}" }
19
- end
20
-
21
- non_uniq = combinations.group_by { |str| str }.select { |_, v| v.count > 1 }
22
-
23
- assert_empty non_uniq.keys
24
- end
25
- end
@@ -1,55 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_8 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.8'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- assertion: [Assertion::Lookahead].flatten,
13
- backref: [:number],
14
- escape: [
15
- Escape::Basic + Escape::ASCII + Escape::Meta + Escape::Control
16
- ].flatten,
17
- group: [Group::V1_8_6].flatten,
18
- quantifier: [
19
- Quantifier::Greedy + Quantifier::Reluctant +
20
- Quantifier::Interval + Quantifier::IntervalReluctant
21
- ].flatten,
22
- },
23
-
24
- excludes: {
25
- assertion: [Assertion::Lookbehind].flatten,
26
-
27
- backref: [
28
- Backreference::All - [:number] + SubexpressionCall::All
29
- ].flatten,
30
-
31
- quantifier: [
32
- Quantifier::Possessive
33
- ].flatten
34
- },
35
- }
36
-
37
- tests.each do |method, types|
38
- expected = method == :excludes ? false : true
39
-
40
- types.each do |type, tokens|
41
- if tokens.nil? or tokens.empty?
42
- define_method "test_syntax_V1_8_#{method}_#{type}" do
43
- assert_equal expected, @syntax.implements?(type, nil)
44
- end
45
- else
46
- tokens.each do |token|
47
- define_method "test_syntax_V1_8_#{method}_#{type}_#{token}" do
48
- assert_equal expected, @syntax.implements?(type, token)
49
- end
50
- end
51
- end
52
- end
53
- end
54
-
55
- end
@@ -1,36 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_1 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.1'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- escape: [
13
- Escape::Hex + Escape::Octal + Escape::Unicode
14
- ].flatten,
15
-
16
- type: [
17
- CharacterType::Hex
18
- ].flatten,
19
-
20
- quantifier: [
21
- Quantifier::Greedy + Quantifier::Reluctant + Quantifier::Possessive
22
- ].flatten,
23
- },
24
- }
25
-
26
- tests.each do |method, types|
27
- types.each do |type, tokens|
28
- tokens.each do |token|
29
- define_method "test_syntax_V1_9_1_#{method}_#{type}_#{token}" do
30
- assert_equal true, @syntax.implements?(type, token)
31
- end
32
- end
33
- end
34
- end
35
-
36
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_3 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.3'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V1_9_3_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,37 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_0_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.0.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Age_V2_0_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Age_V2_0_0
18
- ].flatten,
19
- },
20
- excludes: {
21
- property: [:newline],
22
- nonproperty: [:newline],
23
- }
24
- }
25
-
26
- tests.each do |method, types|
27
- expected = method == :excludes ? false : true
28
-
29
- types.each do |type, tokens|
30
- tokens.each do |token|
31
- define_method "test_syntax_ruby_V2_0_0_#{method}_#{type}_#{token}" do
32
- assert_equal expected, @syntax.implements?(type, token)
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_2_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.2.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V2_2_0_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,129 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxAliases < Test::Unit::TestCase
4
- # 1.8 syntax
5
- def test_syntax_alias_1_8_6
6
- syntax = Regexp::Syntax.new 'ruby/1.8.6'
7
-
8
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
9
- end
10
-
11
- def test_syntax_alias_1_8_alias
12
- syntax = Regexp::Syntax.new 'ruby/1.8'
13
-
14
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
15
- end
16
-
17
- # 1.9 syntax
18
- def test_syntax_alias_1_9_1
19
- syntax = Regexp::Syntax.new 'ruby/1.9.1'
20
-
21
- assert syntax.is_a?(Regexp::Syntax::V1_9_1)
22
- end
23
-
24
- def test_syntax_alias_1_9_alias
25
- syntax = Regexp::Syntax.new 'ruby/1.9'
26
-
27
- assert syntax.is_a?(Regexp::Syntax::V1_9_3)
28
- end
29
-
30
- # 2.0 syntax
31
- def test_syntax_alias_2_0_0
32
- syntax = Regexp::Syntax.new 'ruby/2.0.0'
33
-
34
- assert syntax.is_a?(Regexp::Syntax::V1_9)
35
- end
36
-
37
- def test_syntax_alias_2_0_alias
38
- syntax = Regexp::Syntax.new 'ruby/2.0'
39
-
40
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
41
- end
42
-
43
- # 2.1 syntax
44
- def test_syntax_alias_2_1_alias
45
- syntax = Regexp::Syntax.new 'ruby/2.1'
46
-
47
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
48
- end
49
-
50
- # 2.2 syntax
51
- def test_syntax_alias_2_2_0
52
- syntax = Regexp::Syntax.new 'ruby/2.2.0'
53
-
54
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
55
- end
56
-
57
- def test_syntax_alias_2_2_10
58
- syntax = Regexp::Syntax.new 'ruby/2.2.10'
59
-
60
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
61
- end
62
-
63
- def test_syntax_alias_2_2_alias
64
- syntax = Regexp::Syntax.new 'ruby/2.2'
65
-
66
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
67
- end
68
-
69
- # 2.3 syntax
70
- def test_syntax_alias_2_3_0
71
- syntax = Regexp::Syntax.new 'ruby/2.3.0'
72
-
73
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
74
- end
75
-
76
- def test_syntax_alias_2_3
77
- syntax = Regexp::Syntax.new 'ruby/2.3'
78
-
79
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
80
- end
81
-
82
- # 2.4 syntax
83
- def test_syntax_alias_2_4_0
84
- syntax = Regexp::Syntax.new 'ruby/2.4.0'
85
-
86
- assert syntax.is_a?(Regexp::Syntax::V2_4_0)
87
- end
88
-
89
- def test_syntax_alias_2_4_1
90
- syntax = Regexp::Syntax.new 'ruby/2.4.1'
91
-
92
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
93
- end
94
-
95
- # 2.5 syntax
96
- def test_syntax_alias_2_5_0
97
- syntax = Regexp::Syntax.new 'ruby/2.5.0'
98
-
99
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
100
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
101
- end
102
-
103
- def test_syntax_alias_2_5
104
- syntax = Regexp::Syntax.new 'ruby/2.5'
105
-
106
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
107
- end
108
-
109
- # 2.6 syntax
110
- def test_syntax_alias_2_6_0
111
- syntax = Regexp::Syntax.new 'ruby/2.6.0'
112
-
113
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
114
- assert syntax.is_a?(Regexp::Syntax::V2_6_0)
115
- end
116
-
117
- def test_syntax_alias_2_6
118
- syntax = Regexp::Syntax.new 'ruby/2.6'
119
-
120
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
121
- end
122
-
123
- def test_future_alias_warning
124
- _, stderr_output = capture_output { Regexp::Syntax.new 'ruby/5.0' }
125
-
126
- assert_match(/This library .* but you are running .* \(feature set of .*\)/,
127
- stderr_output)
128
- end
129
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- %w{aliases 1.8 1.9.1 1.9.3 2.0.0 2.2.0}.each do|tc|
4
- require File.expand_path("../test_#{tc}", __FILE__)
5
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../helpers", __FILE__)
2
-
3
- %w{ scanner syntax token lexer parser expression }.each do |subject|
4
- require File.expand_path("../#{subject}/test_all", __FILE__)
5
- end
@@ -1,2 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
- require File.expand_path("../test_token", __FILE__)
@@ -1,107 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
-
3
- class Token < Test::Unit::TestCase
4
-
5
- def test_token_offset
6
- regexp = /ab?cd/
7
- tokens = RL.lex(regexp)
8
-
9
- assert_equal 'b', tokens[1].text
10
- assert_equal [1, 2], tokens[1].offset
11
-
12
- assert_equal '?', tokens[2].text
13
- assert_equal [2, 3], tokens[2].offset
14
-
15
- assert_equal 'cd', tokens[3].text
16
- assert_equal [3, 5], tokens[3].offset
17
- end
18
-
19
- def test_token_length
20
- regexp = /abc?def/
21
- tokens = RL.lex(regexp)
22
-
23
- assert_equal 'ab', tokens[0].text
24
- assert_equal 2, tokens[0].length
25
-
26
- assert_equal 'c', tokens[1].text
27
- assert_equal 1, tokens[1].length
28
-
29
- assert_equal '?', tokens[2].text
30
- assert_equal 1, tokens[2].length
31
-
32
- assert_equal 'def', tokens[3].text
33
- assert_equal 3, tokens[3].length
34
- end
35
-
36
- def test_token_to_h
37
- regexp = /abc?def/
38
- tokens = RL.lex(regexp)
39
-
40
- assert_equal 'ab', tokens[0].text
41
- assert_equal({
42
- :type => :literal,
43
- :token => :literal,
44
- :text => 'ab',
45
- :ts => 0,
46
- :te => 2,
47
- :level => 0,
48
- :set_level => 0,
49
- :conditional_level => 0
50
- }, tokens[0].to_h
51
- )
52
-
53
- assert_equal '?', tokens[2].text
54
- assert_equal({
55
- :type => :quantifier,
56
- :token => :zero_or_one,
57
- :text => '?',
58
- :ts => 3,
59
- :te => 4,
60
- :level => 0,
61
- :set_level => 0,
62
- :conditional_level => 0
63
- }, tokens[2].to_h
64
- )
65
- end
66
-
67
- def test_token_next
68
- regexp = /a+b?c*d{2,3}/
69
- tokens = RL.lex(regexp)
70
-
71
- a = tokens.first
72
- assert_equal 'a', a.text
73
-
74
- plus = a.next
75
- assert_equal '+', plus.text
76
-
77
- b = plus.next
78
- assert_equal 'b', b.text
79
-
80
- interval = tokens.last
81
- assert_equal '{2,3}', interval.text
82
-
83
- assert_equal nil, interval.next
84
- end
85
-
86
- def test_token_previous
87
- regexp = /a+b?c*d{2,3}/
88
- tokens = RL.lex(regexp)
89
-
90
- interval = tokens.last
91
- assert_equal '{2,3}', interval.text
92
-
93
- d = interval.previous
94
- assert_equal 'd', d.text
95
-
96
- star = d.previous
97
- assert_equal '*', star.text
98
-
99
- c = star.previous
100
- assert_equal 'c', c.text
101
-
102
- a = tokens.first
103
- assert_equal 'a', a.text
104
- assert_equal nil, a.previous
105
- end
106
-
107
- end