regexp_parser 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +6 -0
- data/lib/regexp_parser/syntax/ruby/1.9.1.rb +1 -1
- data/lib/regexp_parser/syntax/tokens/escape.rb +2 -0
- data/lib/regexp_parser/version.rb +1 -1
- data/test/expression/test_base.rb +32 -34
- data/test/expression/test_clone.rb +49 -47
- data/test/expression/test_conditionals.rb +40 -40
- data/test/expression/test_free_space.rb +4 -2
- data/test/expression/test_set.rb +16 -16
- data/test/expression/test_strfregexp.rb +74 -74
- data/test/expression/test_subexpression.rb +2 -2
- data/test/expression/test_tests.rb +57 -57
- data/test/expression/test_to_h.rb +11 -6
- data/test/expression/test_to_s.rb +22 -15
- data/test/expression/test_traverse.rb +26 -29
- data/test/lexer/test_all.rb +9 -7
- data/test/lexer/test_conditionals.rb +35 -11
- data/test/lexer/test_keep.rb +6 -6
- data/test/lexer/test_literals.rb +20 -10
- data/test/lexer/test_nesting.rb +14 -7
- data/test/lexer/test_refcalls.rb +12 -5
- data/test/parser/test_all.rb +15 -13
- data/test/parser/test_alternation.rb +29 -26
- data/test/parser/test_anchors.rb +7 -8
- data/test/parser/test_conditionals.rb +43 -41
- data/test/parser/test_escapes.rb +18 -16
- data/test/parser/test_free_space.rb +33 -33
- data/test/parser/test_groups.rb +46 -46
- data/test/parser/test_keep.rb +4 -4
- data/test/parser/test_properties.rb +18 -16
- data/test/parser/test_quantifiers.rb +184 -163
- data/test/parser/test_refcalls.rb +48 -32
- data/test/parser/test_sets.rb +102 -88
- data/test/parser/test_types.rb +7 -8
- data/test/scanner/test_all.rb +6 -4
- data/test/scanner/test_anchors.rb +8 -5
- data/test/scanner/test_conditionals.rb +38 -20
- data/test/scanner/test_escapes.rb +8 -6
- data/test/scanner/test_free_space.rb +89 -65
- data/test/scanner/test_groups.rb +27 -32
- data/test/scanner/test_keep.rb +24 -22
- data/test/scanner/test_literals.rb +11 -7
- data/test/scanner/test_meta.rb +11 -7
- data/test/scanner/test_properties.rb +16 -14
- data/test/scanner/test_quantifiers.rb +8 -9
- data/test/scanner/test_refcalls.rb +26 -23
- data/test/scanner/test_scripts.rb +11 -10
- data/test/scanner/test_sets.rb +8 -5
- data/test/scanner/test_types.rb +17 -15
- data/test/scanner/test_unicode_blocks.rb +11 -10
- data/test/syntax/ruby/test_1.8.rb +4 -8
- data/test/syntax/ruby/test_1.9.1.rb +1 -7
- data/test/syntax/ruby/test_1.9.3.rb +3 -7
- data/test/syntax/ruby/test_2.0.0.rb +1 -7
- data/test/syntax/ruby/test_2.2.0.rb +1 -7
- data/test/token/test_token.rb +29 -31
- metadata +3 -3
@@ -3,9 +3,11 @@ require File.expand_path("../../helpers", __FILE__)
|
|
3
3
|
class ExpressionToH < Test::Unit::TestCase
|
4
4
|
|
5
5
|
def test_expression_to_h
|
6
|
-
|
6
|
+
root = RP.parse('abc')
|
7
7
|
|
8
|
-
|
8
|
+
hash = root.to_h
|
9
|
+
|
10
|
+
assert_equal hash, {
|
9
11
|
:token => :root,
|
10
12
|
:type => :expression,
|
11
13
|
:text => 'abc',
|
@@ -30,19 +32,22 @@ class ExpressionToH < Test::Unit::TestCase
|
|
30
32
|
:conditional_level => 0
|
31
33
|
}
|
32
34
|
]
|
33
|
-
}
|
35
|
+
}
|
34
36
|
end
|
35
37
|
|
36
38
|
def test_expression_quantifier_to_h
|
37
|
-
|
39
|
+
root = RP.parse('a{2,4}')
|
40
|
+
exp = root.expressions.at(0)
|
41
|
+
|
42
|
+
hash = exp.quantifier.to_h
|
38
43
|
|
39
|
-
assert_equal
|
44
|
+
assert_equal hash, {
|
40
45
|
:max => 4,
|
41
46
|
:min => 2,
|
42
47
|
:mode => :greedy,
|
43
48
|
:text => '{2,4}',
|
44
49
|
:token => :interval
|
45
|
-
}
|
50
|
+
}
|
46
51
|
end
|
47
52
|
|
48
53
|
end
|
@@ -4,48 +4,57 @@ class ExpressionToS < Test::Unit::TestCase
|
|
4
4
|
|
5
5
|
def test_expression_to_s_literal_alternation
|
6
6
|
pattern = 'abcd|ghij|klmn|pqur'
|
7
|
-
|
7
|
+
|
8
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
8
9
|
end
|
9
10
|
|
10
11
|
def test_expression_to_s_quantified_alternations
|
11
12
|
pattern = '(?:a?[b]+(c){2}|d+[e]*(f)?)|(?:g+[h]?(i){2,3}|j*[k]{3,5}(l)?)'
|
12
|
-
|
13
|
+
|
14
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
13
15
|
end
|
14
16
|
|
15
17
|
def test_expression_to_s_quantified_sets
|
16
18
|
pattern = '[abc]+|[^def]{3,6}'
|
17
|
-
|
19
|
+
|
20
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
18
21
|
end
|
19
22
|
|
20
23
|
def test_expression_to_s_property_sets
|
21
24
|
pattern = '[\a\b\p{Lu}\P{Z}\c\d]+'
|
22
|
-
|
25
|
+
|
26
|
+
assert_equal pattern, RP.parse(pattern, 'ruby/1.9').to_s
|
23
27
|
end
|
24
28
|
|
25
29
|
def test_expression_to_s_groups
|
26
30
|
pattern = "(a(?>b(?:c(?<n>d(?'N'e)??f)+g)*+h)*i)++"
|
27
|
-
|
31
|
+
|
32
|
+
assert_equal pattern, RP.parse(pattern, 'ruby/1.9').to_s
|
28
33
|
end
|
29
34
|
|
30
35
|
def test_expression_to_s_assertions
|
31
36
|
pattern = '(a+(?=b+(?!c+(?<=d+(?<!e+)?f+)?g+)?h+)?i+)?'
|
32
|
-
|
37
|
+
|
38
|
+
assert_equal pattern, RP.parse(pattern, 'ruby/1.9').to_s
|
33
39
|
end
|
34
40
|
|
35
41
|
def test_expression_to_s_comments
|
36
42
|
pattern = '(?#start)a(?#middle)b(?#end)'
|
37
|
-
|
43
|
+
|
44
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
38
45
|
end
|
39
46
|
|
40
47
|
def test_expression_to_s_options
|
41
48
|
pattern = '(?mix:start)a(?-mix:middle)b(?i-mx:end)'
|
42
|
-
|
49
|
+
|
50
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
43
51
|
end
|
44
52
|
|
45
53
|
def test_expression_to_s_url
|
46
54
|
pattern = '(^$)|(^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*'+
|
47
55
|
'\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$)'
|
48
|
-
|
56
|
+
|
57
|
+
assert_equal pattern, RP.parse(pattern).to_s
|
49
58
|
end
|
50
59
|
|
51
60
|
def test_expression_to_s_multiline_source
|
@@ -57,7 +66,7 @@ class ExpressionToS < Test::Unit::TestCase
|
|
57
66
|
\z
|
58
67
|
}x
|
59
68
|
|
60
|
-
assert_equal
|
69
|
+
assert_equal multiline.source, RP.parse(multiline).to_s
|
61
70
|
end
|
62
71
|
|
63
72
|
def test_expression_to_s_multiline_to_s
|
@@ -69,7 +78,7 @@ class ExpressionToS < Test::Unit::TestCase
|
|
69
78
|
\z
|
70
79
|
}x
|
71
80
|
|
72
|
-
assert_equal
|
81
|
+
assert_equal multiline.to_s, RP.parse(multiline.to_s).to_s
|
73
82
|
end
|
74
83
|
|
75
84
|
# Free spacing expressions that use spaces between quantifiers and their
|
@@ -88,10 +97,8 @@ class ExpressionToS < Test::Unit::TestCase
|
|
88
97
|
str = 'bbbcged'
|
89
98
|
root = RP.parse(multiline)
|
90
99
|
|
91
|
-
assert_equal(
|
92
|
-
|
93
|
-
Regexp.new(root.to_s, Regexp::EXTENDED).match(str)[0]
|
94
|
-
)
|
100
|
+
assert_equal multiline.match(str)[0],
|
101
|
+
Regexp.new(root.to_s, Regexp::EXTENDED).match(str)[0]
|
95
102
|
end
|
96
103
|
|
97
104
|
end
|
@@ -15,10 +15,10 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
15
15
|
exits += 1 if event == :exit
|
16
16
|
}
|
17
17
|
|
18
|
-
assert_equal
|
19
|
-
assert_equal
|
18
|
+
assert_equal 7, enters
|
19
|
+
assert_equal exits, enters
|
20
20
|
|
21
|
-
assert_equal
|
21
|
+
assert_equal 8, visits
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_subexpression_traverse_include_self
|
@@ -34,16 +34,16 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
34
34
|
exits += 1 if event == :exit
|
35
35
|
}
|
36
36
|
|
37
|
-
assert_equal
|
38
|
-
assert_equal
|
37
|
+
assert_equal 8, enters
|
38
|
+
assert_equal exits, enters
|
39
39
|
|
40
|
-
assert_equal
|
40
|
+
assert_equal 8, visits
|
41
41
|
end
|
42
42
|
|
43
43
|
def test_subexpression_walk_alias
|
44
44
|
root = RP.parse(/abc/)
|
45
45
|
|
46
|
-
assert_equal
|
46
|
+
assert_equal true, root.respond_to?(:walk)
|
47
47
|
end
|
48
48
|
|
49
49
|
def test_subexpression_each_expression
|
@@ -54,7 +54,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
54
54
|
count += 1
|
55
55
|
}
|
56
56
|
|
57
|
-
assert_equal
|
57
|
+
assert_equal 10, count
|
58
58
|
end
|
59
59
|
|
60
60
|
def test_subexpression_each_expression_include_self
|
@@ -65,7 +65,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
65
65
|
count += 1
|
66
66
|
}
|
67
67
|
|
68
|
-
assert_equal
|
68
|
+
assert_equal 11, count
|
69
69
|
end
|
70
70
|
|
71
71
|
def test_subexpression_each_expression_indices
|
@@ -74,7 +74,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
74
74
|
indices = []
|
75
75
|
root.each_expression {|exp, index| indices << index}
|
76
76
|
|
77
|
-
assert_equal
|
77
|
+
assert_equal [0, 1, 0, 2], indices
|
78
78
|
end
|
79
79
|
|
80
80
|
def test_subexpression_each_expression_indices_include_self
|
@@ -83,7 +83,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
83
83
|
indices = []
|
84
84
|
root.each_expression(true) {|exp, index| indices << index}
|
85
85
|
|
86
|
-
assert_equal
|
86
|
+
assert_equal [0, 0, 1, 0, 2], indices
|
87
87
|
end
|
88
88
|
|
89
89
|
def test_subexpression_map_without_block
|
@@ -91,14 +91,14 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
91
91
|
|
92
92
|
array = root.map
|
93
93
|
|
94
|
-
assert_equal
|
95
|
-
assert_equal
|
94
|
+
assert_equal Array, array.class
|
95
|
+
assert_equal 5, array.length
|
96
96
|
|
97
97
|
array.each do |item|
|
98
|
-
assert_equal
|
99
|
-
assert_equal
|
100
|
-
assert_equal
|
101
|
-
assert_equal
|
98
|
+
assert_equal Array, item.class
|
99
|
+
assert_equal 2, item.length
|
100
|
+
assert_equal true, item.first.is_a?(Regexp::Expression::Base)
|
101
|
+
assert_equal true, item.last.is_a?(Fixnum)
|
102
102
|
end
|
103
103
|
end
|
104
104
|
|
@@ -107,8 +107,8 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
107
107
|
|
108
108
|
array = root.map(true)
|
109
109
|
|
110
|
-
assert_equal
|
111
|
-
assert_equal
|
110
|
+
assert_equal Array, array.class
|
111
|
+
assert_equal 6, array.length
|
112
112
|
end
|
113
113
|
|
114
114
|
def test_subexpression_map_indices
|
@@ -116,7 +116,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
116
116
|
|
117
117
|
indices = root.map {|exp, index| index}
|
118
118
|
|
119
|
-
assert_equal
|
119
|
+
assert_equal [0, 1, 0, 1, 0, 2, 3], indices
|
120
120
|
end
|
121
121
|
|
122
122
|
def test_subexpression_map_indices_include_self
|
@@ -124,7 +124,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
124
124
|
|
125
125
|
indices = root.map(true) {|exp, index| index}
|
126
126
|
|
127
|
-
assert_equal
|
127
|
+
assert_equal [0, 0, 1, 0, 1, 0, 2, 3], indices
|
128
128
|
end
|
129
129
|
|
130
130
|
def test_subexpression_map_expressions
|
@@ -134,10 +134,9 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
134
134
|
[exp.level, exp.text] if exp.terminal?
|
135
135
|
}.compact
|
136
136
|
|
137
|
-
assert_equal
|
138
|
-
|
139
|
-
levels
|
140
|
-
)
|
137
|
+
assert_equal [
|
138
|
+
[0, 'a'], [1, 'b'], [2, 'c'], [3, 'd']
|
139
|
+
], levels
|
141
140
|
end
|
142
141
|
|
143
142
|
def test_subexpression_map_expressions_include_self
|
@@ -147,7 +146,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
147
146
|
[exp.level, exp.to_s]
|
148
147
|
}.compact
|
149
148
|
|
150
|
-
assert_equal
|
149
|
+
assert_equal [
|
151
150
|
[nil, 'a(b(c(d)))'],
|
152
151
|
[0, 'a'],
|
153
152
|
[0, '(b(c(d)))'],
|
@@ -156,9 +155,7 @@ class SubexpressionTraverse < Test::Unit::TestCase
|
|
156
155
|
[2, 'c'],
|
157
156
|
[2, '(d)'],
|
158
157
|
[3, 'd']
|
159
|
-
],
|
160
|
-
levels
|
161
|
-
)
|
158
|
+
], levels
|
162
159
|
end
|
163
160
|
|
164
161
|
end
|
data/test/lexer/test_all.rb
CHANGED
@@ -15,25 +15,27 @@ end
|
|
15
15
|
class TestRegexpLexer < Test::Unit::TestCase
|
16
16
|
|
17
17
|
def test_lexer_returns_an_array
|
18
|
-
assert_instance_of
|
18
|
+
assert_instance_of Array, RL.lex('abc')
|
19
19
|
end
|
20
20
|
|
21
21
|
def test_lexer_returns_tokens
|
22
22
|
tokens = RL.lex('^abc+[^one]{2,3}\b\d\\\C-C$')
|
23
|
-
assert( tokens.all?{|token| token.kind_of?(Regexp::Token)},
|
24
|
-
"Not all array members are tokens")
|
25
23
|
|
26
|
-
assert
|
27
|
-
"Not all
|
24
|
+
assert tokens.all?{ |token| token.kind_of?(Regexp::Token) },
|
25
|
+
"Not all array members are tokens"
|
26
|
+
|
27
|
+
assert tokens.all?{ |token| token.to_a.length == 8 },
|
28
|
+
"Not all tokens have a length of 8"
|
28
29
|
end
|
29
30
|
|
30
31
|
def test_lexer_token_count
|
31
32
|
tokens = RL.lex(/^(one|two){2,3}([^d\]efm-qz\,\-]*)(ghi)+$/i)
|
32
|
-
|
33
|
+
|
34
|
+
assert_equal 26, tokens.length
|
33
35
|
end
|
34
36
|
|
35
37
|
def test_lexer_scan_alias
|
36
|
-
assert_equal
|
38
|
+
assert_equal RL.lex(/a|b|c/), RL.scan(/a|b|c/)
|
37
39
|
end
|
38
40
|
|
39
41
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
require File.expand_path("../../helpers", __FILE__)
|
2
2
|
|
3
3
|
class LexerConditionals < Test::Unit::TestCase
|
4
|
-
|
5
4
|
if RUBY_VERSION >= '2.0'
|
6
5
|
|
7
6
|
# Basic lexer output and nesting tests
|
@@ -12,11 +11,19 @@ class LexerConditionals < Test::Unit::TestCase
|
|
12
11
|
'(?<D>a)(?(<D>)b|c)' => [8, :conditional, :close, ')', 17, 18, 0, 0, 0],
|
13
12
|
}
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
define_method "test_lexer_#{test[1]}_#{test[2]}_#{count+=1}" do
|
14
|
+
tests.each_with_index do |(pattern, (index, type, token, text, ts, te, level, set_level, conditional_level)), count|
|
15
|
+
define_method "test_lexer_#{type}_#{token}_#{count}" do
|
18
16
|
tokens = RL.lex(pattern)
|
19
|
-
|
17
|
+
struct = tokens.at(index)
|
18
|
+
|
19
|
+
assert_equal type, struct.type
|
20
|
+
assert_equal token, struct.token
|
21
|
+
assert_equal text, struct.text
|
22
|
+
assert_equal ts, struct.ts
|
23
|
+
assert_equal te, struct.te
|
24
|
+
assert_equal level, struct.level
|
25
|
+
assert_equal set_level, struct.set_level
|
26
|
+
assert_equal conditional_level, struct.conditional_level
|
20
27
|
end
|
21
28
|
end
|
22
29
|
|
@@ -45,8 +52,17 @@ class LexerConditionals < Test::Unit::TestCase
|
|
45
52
|
|
46
53
|
[22, :group, :close, ')', 44, 45, 1, 0, 0],
|
47
54
|
[23, :group, :close, ')', 45, 46, 0, 0, 0]
|
48
|
-
].each do |
|
49
|
-
|
55
|
+
].each do |index, type, token, text, ts, te, level, set_level, conditional_level|
|
56
|
+
struct = tokens.at(index)
|
57
|
+
|
58
|
+
assert_equal type, struct.type
|
59
|
+
assert_equal token, struct.token
|
60
|
+
assert_equal text, struct.text
|
61
|
+
assert_equal ts, struct.ts
|
62
|
+
assert_equal te, struct.te
|
63
|
+
assert_equal level, struct.level
|
64
|
+
assert_equal set_level, struct.set_level
|
65
|
+
assert_equal conditional_level, struct.conditional_level
|
50
66
|
end
|
51
67
|
end
|
52
68
|
|
@@ -91,11 +107,19 @@ class LexerConditionals < Test::Unit::TestCase
|
|
91
107
|
[35, :conditional, :close, ')', 53, 54, 0, 0, 2],
|
92
108
|
[36, :conditional, :close, ')', 54, 55, 0, 0, 1],
|
93
109
|
[37, :conditional, :close, ')', 55, 56, 0, 0, 0]
|
94
|
-
].each do |
|
95
|
-
|
110
|
+
].each do |index, type, token, text, ts, te, level, set_level, conditional_level|
|
111
|
+
struct = tokens.at(index)
|
112
|
+
|
113
|
+
assert_equal type, struct.type
|
114
|
+
assert_equal token, struct.token
|
115
|
+
assert_equal text, struct.text
|
116
|
+
assert_equal ts, struct.ts
|
117
|
+
assert_equal te, struct.te
|
118
|
+
assert_equal level, struct.level
|
119
|
+
assert_equal set_level, struct.set_level
|
120
|
+
assert_equal conditional_level, struct.conditional_level
|
96
121
|
end
|
97
122
|
end
|
98
123
|
|
99
|
-
end
|
100
|
-
|
124
|
+
end # if RUBY_VERSION >= '2.0'
|
101
125
|
end
|
data/test/lexer/test_keep.rb
CHANGED
@@ -6,19 +6,19 @@ class LexerKeep < Test::Unit::TestCase
|
|
6
6
|
regexp = /ab\Kcd/
|
7
7
|
tokens = RL.lex(regexp)
|
8
8
|
|
9
|
-
assert_equal
|
10
|
-
assert_equal
|
9
|
+
assert_equal :keep, tokens[1].type
|
10
|
+
assert_equal :mark, tokens[1].token
|
11
11
|
end
|
12
12
|
|
13
13
|
def test_lex_keep_nested
|
14
14
|
regexp = /(a\Kb)|(c\\\Kd)ef/
|
15
15
|
tokens = RL.lex(regexp)
|
16
16
|
|
17
|
-
assert_equal
|
18
|
-
assert_equal
|
17
|
+
assert_equal :keep, tokens[2].type
|
18
|
+
assert_equal :mark, tokens[2].token
|
19
19
|
|
20
|
-
assert_equal
|
21
|
-
assert_equal
|
20
|
+
assert_equal :keep, tokens[9].type
|
21
|
+
assert_equal :mark, tokens[9].token
|
22
22
|
end
|
23
23
|
|
24
24
|
end
|
data/test/lexer/test_literals.rb
CHANGED
@@ -90,31 +90,41 @@ class LexerLiterals < Test::Unit::TestCase
|
|
90
90
|
},
|
91
91
|
}
|
92
92
|
|
93
|
-
|
94
|
-
|
95
|
-
define_method "test_lex_literal_runs_#{count+=1}" do
|
96
|
-
|
93
|
+
tests.each_with_index do |(pattern, checks), count|
|
94
|
+
define_method "test_lex_literal_runs_#{count}" do
|
97
95
|
tokens = RL.lex(pattern)
|
98
|
-
checks.each do |offset, token|
|
99
|
-
assert_equal( token, tokens[offset].to_a )
|
100
|
-
end
|
101
96
|
|
97
|
+
checks.each do |index, (type, token, text, ts, te, level, set_level, conditional_level)|
|
98
|
+
struct = tokens.at(index)
|
99
|
+
|
100
|
+
assert_equal type, struct.type
|
101
|
+
assert_equal token, struct.token
|
102
|
+
assert_equal text, struct.text
|
103
|
+
assert_equal ts, struct.ts
|
104
|
+
assert_equal te, struct.te
|
105
|
+
assert_equal level, struct.level
|
106
|
+
assert_equal set_level, struct.set_level
|
107
|
+
assert_equal conditional_level, struct.conditional_level
|
108
|
+
end
|
102
109
|
end
|
103
110
|
end
|
104
111
|
|
105
112
|
def test_lex_single_2_byte_char
|
106
113
|
tokens = RL.lex('ا+')
|
107
|
-
|
114
|
+
|
115
|
+
assert_equal 2, tokens.length
|
108
116
|
end
|
109
117
|
|
110
118
|
def test_lex_single_3_byte_char
|
111
119
|
tokens = RL.lex('れ+')
|
112
|
-
|
120
|
+
|
121
|
+
assert_equal 2, tokens.length
|
113
122
|
end
|
114
123
|
|
115
124
|
def test_lex_single_4_byte_char
|
116
125
|
tokens = RL.lex('𝄞+')
|
117
|
-
|
126
|
+
|
127
|
+
assert_equal 2, tokens.length
|
118
128
|
end
|
119
129
|
|
120
130
|
end
|