regexp_parser 1.5.0 → 1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/lib/regexp_parser/expression.rb +6 -43
- data/lib/regexp_parser/expression/classes/conditional.rb +3 -2
- data/lib/regexp_parser/expression/classes/escape.rb +0 -4
- data/lib/regexp_parser/expression/methods/match.rb +13 -0
- data/lib/regexp_parser/expression/methods/options.rb +35 -0
- data/lib/regexp_parser/expression/methods/strfregexp.rb +0 -1
- data/lib/regexp_parser/expression/methods/tests.rb +6 -15
- data/lib/regexp_parser/expression/sequence.rb +3 -2
- data/lib/regexp_parser/expression/sequence_operation.rb +2 -6
- data/lib/regexp_parser/lexer.rb +0 -21
- data/lib/regexp_parser/parser.rb +22 -21
- data/lib/regexp_parser/scanner.rb +1159 -1329
- data/lib/regexp_parser/scanner/char_type.rl +0 -3
- data/lib/regexp_parser/scanner/scanner.rl +82 -190
- data/lib/regexp_parser/version.rb +1 -1
- data/spec/expression/base_spec.rb +14 -0
- data/spec/expression/methods/match_length_spec.rb +13 -0
- data/spec/expression/methods/match_spec.rb +25 -0
- data/spec/expression/methods/tests_spec.rb +2 -0
- data/spec/expression/options_spec.rb +128 -0
- data/spec/expression/root_spec.rb +9 -0
- data/spec/expression/sequence_spec.rb +9 -0
- data/spec/lexer/conditionals_spec.rb +49 -119
- data/spec/lexer/escapes_spec.rb +8 -32
- data/spec/lexer/keep_spec.rb +5 -17
- data/spec/lexer/literals_spec.rb +73 -110
- data/spec/lexer/nesting_spec.rb +86 -117
- data/spec/lexer/refcalls_spec.rb +51 -50
- data/spec/parser/all_spec.rb +13 -1
- data/spec/parser/anchors_spec.rb +9 -23
- data/spec/parser/conditionals_spec.rb +9 -9
- data/spec/parser/errors_spec.rb +22 -43
- data/spec/parser/escapes_spec.rb +33 -44
- data/spec/parser/groups_spec.rb +98 -257
- data/spec/parser/keep_spec.rb +2 -15
- data/spec/parser/posix_classes_spec.rb +5 -24
- data/spec/parser/properties_spec.rb +42 -54
- data/spec/parser/quantifiers_spec.rb +41 -283
- data/spec/parser/refcalls_spec.rb +60 -185
- data/spec/parser/set/intersections_spec.rb +17 -17
- data/spec/parser/set/ranges_spec.rb +17 -17
- data/spec/parser/sets_spec.rb +5 -5
- data/spec/parser/types_spec.rb +11 -36
- data/spec/scanner/anchors_spec.rb +13 -28
- data/spec/scanner/conditionals_spec.rb +121 -173
- data/spec/scanner/errors_spec.rb +65 -87
- data/spec/scanner/escapes_spec.rb +49 -50
- data/spec/scanner/free_space_spec.rb +102 -165
- data/spec/scanner/groups_spec.rb +45 -64
- data/spec/scanner/keep_spec.rb +5 -28
- data/spec/scanner/literals_spec.rb +45 -81
- data/spec/scanner/meta_spec.rb +13 -33
- data/spec/scanner/properties_spec.rb +43 -286
- data/spec/scanner/quantifiers_spec.rb +13 -28
- data/spec/scanner/refcalls_spec.rb +32 -48
- data/spec/scanner/sets_spec.rb +88 -102
- data/spec/scanner/types_spec.rb +10 -25
- data/spec/spec_helper.rb +1 -0
- data/spec/support/shared_examples.rb +77 -0
- data/spec/syntax/syntax_spec.rb +4 -0
- data/spec/syntax/versions/1.8.6_spec.rb +12 -33
- data/spec/syntax/versions/1.9.1_spec.rb +5 -18
- data/spec/syntax/versions/1.9.3_spec.rb +4 -17
- data/spec/syntax/versions/2.0.0_spec.rb +8 -23
- data/spec/syntax/versions/2.2.0_spec.rb +4 -17
- data/spec/syntax/versions/aliases_spec.rb +25 -109
- metadata +14 -6
- data/spec/scanner/scripts_spec.rb +0 -49
- data/spec/scanner/unicode_blocks_spec.rb +0 -28
data/spec/scanner/keep_spec.rb
CHANGED
@@ -1,33 +1,10 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe('Keep scanning') do
|
4
|
-
|
5
|
-
|
6
|
-
result = tokens.at(1)
|
4
|
+
include_examples 'scan', /ab\Kcd/,
|
5
|
+
1 => [:keep, :mark, '\K', 2, 4]
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
expect(result[3]).to eq 2
|
12
|
-
expect(result[4]).to eq 4
|
13
|
-
end
|
14
|
-
|
15
|
-
specify('scan keep nested') do
|
16
|
-
tokens = RS.scan(/(a\Kb)|(c\\\Kd)ef/)
|
17
|
-
|
18
|
-
first = tokens.at(2)
|
19
|
-
second = tokens.at(9)
|
20
|
-
|
21
|
-
expect(first[0]).to eq :keep
|
22
|
-
expect(first[1]).to eq :mark
|
23
|
-
expect(first[2]).to eq '\\K'
|
24
|
-
expect(first[3]).to eq 2
|
25
|
-
expect(first[4]).to eq 4
|
26
|
-
|
27
|
-
expect(second[0]).to eq :keep
|
28
|
-
expect(second[1]).to eq :mark
|
29
|
-
expect(second[2]).to eq '\\K'
|
30
|
-
expect(second[3]).to eq 11
|
31
|
-
expect(second[4]).to eq 13
|
32
|
-
end
|
7
|
+
include_examples 'scan', /(a\Kb)|(c\\\Kd)ef/,
|
8
|
+
2 => [:keep, :mark, '\K', 2, 4],
|
9
|
+
9 => [:keep, :mark, '\K', 11, 13]
|
33
10
|
end
|
@@ -1,85 +1,49 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe('UTF8 scanning') do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
},
|
50
|
-
|
51
|
-
# 4 byte wide characters, Osmanya
|
52
|
-
'𐒀𐒁?𐒂ab+𐒃' => {
|
53
|
-
0 => [:literal, :literal, '𐒀𐒁', 0, 8],
|
54
|
-
1 => [:quantifier, :zero_or_one, '?', 8, 9],
|
55
|
-
2 => [:literal, :literal, '𐒂ab', 9, 15],
|
56
|
-
3 => [:quantifier, :one_or_more, '+', 15, 16],
|
57
|
-
4 => [:literal, :literal, '𐒃', 16, 20],
|
58
|
-
},
|
59
|
-
|
60
|
-
'mu𝄞?si*𝄫c+' => {
|
61
|
-
0 => [:literal, :literal, 'mu𝄞', 0, 6],
|
62
|
-
1 => [:quantifier, :zero_or_one, '?', 6, 7],
|
63
|
-
2 => [:literal, :literal, 'si', 7, 9],
|
64
|
-
3 => [:quantifier, :zero_or_more, '*', 9, 10],
|
65
|
-
4 => [:literal, :literal, '𝄫c', 10, 15],
|
66
|
-
5 => [:quantifier, :one_or_more, '+', 15, 16],
|
67
|
-
},
|
68
|
-
}
|
69
|
-
|
70
|
-
tests.each_with_index do |(pattern, checks), count|
|
71
|
-
specify("scanner_utf8_runs_#{count}") do
|
72
|
-
tokens = RS.scan(pattern)
|
73
|
-
|
74
|
-
checks.each do |index, (type, token, text, ts, te)|
|
75
|
-
result = tokens[index]
|
76
|
-
|
77
|
-
expect(result[0]).to eq type
|
78
|
-
expect(result[1]).to eq token
|
79
|
-
expect(result[2]).to eq text
|
80
|
-
expect(result[3]).to eq ts
|
81
|
-
expect(result[4]).to eq te
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
4
|
+
# ascii, single byte characters
|
5
|
+
include_examples 'scan', 'a', 0 => [:literal, :literal, 'a', 0, 1]
|
6
|
+
|
7
|
+
include_examples 'scan', 'ab+', 0 => [:literal, :literal, 'ab', 0, 2]
|
8
|
+
include_examples 'scan', 'ab+', 1 => [:quantifier, :one_or_more, '+', 2, 3]
|
9
|
+
|
10
|
+
# 2 byte wide characters, Arabic
|
11
|
+
include_examples 'scan', 'aاbبcت', 0 => [:literal, :literal, 'aاbبcت', 0, 9]
|
12
|
+
|
13
|
+
include_examples 'scan', 'aاbبت?', 0 => [:literal, :literal, 'aاbبت', 0, 8]
|
14
|
+
include_examples 'scan', 'aاbبت?', 1 => [:quantifier, :zero_or_one, '?', 8, 9]
|
15
|
+
|
16
|
+
include_examples 'scan', 'aا?bبcت+', 0 => [:literal, :literal, 'aا', 0, 3]
|
17
|
+
include_examples 'scan', 'aا?bبcت+', 1 => [:quantifier, :zero_or_one, '?', 3, 4]
|
18
|
+
include_examples 'scan', 'aا?bبcت+', 2 => [:literal, :literal, 'bبcت', 4, 10]
|
19
|
+
include_examples 'scan', 'aا?bبcت+', 3 => [:quantifier, :one_or_more, '+', 10, 11]
|
20
|
+
|
21
|
+
include_examples 'scan', 'a(اbب+)cت?', 0 => [:literal, :literal, 'a', 0, 1]
|
22
|
+
include_examples 'scan', 'a(اbب+)cت?', 1 => [:group, :capture, '(', 1, 2]
|
23
|
+
include_examples 'scan', 'a(اbب+)cت?', 2 => [:literal, :literal, 'اbب', 2, 7]
|
24
|
+
include_examples 'scan', 'a(اbب+)cت?', 3 => [:quantifier, :one_or_more, '+', 7, 8]
|
25
|
+
include_examples 'scan', 'a(اbب+)cت?', 4 => [:group, :close, ')', 8, 9]
|
26
|
+
include_examples 'scan', 'a(اbب+)cت?', 5 => [:literal, :literal, 'cت', 9, 12]
|
27
|
+
include_examples 'scan', 'a(اbب+)cت?', 6 => [:quantifier, :zero_or_one, '?', 12, 13]
|
28
|
+
|
29
|
+
# 3 byte wide characters, Japanese
|
30
|
+
include_examples 'scan', 'ab?れます+cd', 0 => [:literal, :literal, 'ab', 0, 2]
|
31
|
+
include_examples 'scan', 'ab?れます+cd', 1 => [:quantifier, :zero_or_one, '?', 2, 3]
|
32
|
+
include_examples 'scan', 'ab?れます+cd', 2 => [:literal, :literal, 'れます', 3, 12]
|
33
|
+
include_examples 'scan', 'ab?れます+cd', 3 => [:quantifier, :one_or_more, '+', 12, 13]
|
34
|
+
include_examples 'scan', 'ab?れます+cd', 4 => [:literal, :literal, 'cd', 13, 15]
|
35
|
+
|
36
|
+
# 4 byte wide characters, Osmanya
|
37
|
+
include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 0 => [:literal, :literal, '𐒀𐒁', 0, 8]
|
38
|
+
include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 1 => [:quantifier, :zero_or_one, '?', 8, 9]
|
39
|
+
include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 2 => [:literal, :literal, '𐒂ab', 9, 15]
|
40
|
+
include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 3 => [:quantifier, :one_or_more, '+', 15, 16]
|
41
|
+
include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃', 4 => [:literal, :literal, '𐒃', 16, 20]
|
42
|
+
|
43
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 0 => [:literal, :literal, 'mu𝄞', 0, 6]
|
44
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 1 => [:quantifier, :zero_or_one, '?', 6, 7]
|
45
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 2 => [:literal, :literal, 'si', 7, 9]
|
46
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 3 => [:quantifier, :zero_or_more, '*', 9, 10]
|
47
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 4 => [:literal, :literal, '𝄫c', 10, 15]
|
48
|
+
include_examples 'scan', 'mu𝄞?si*𝄫c+', 5 => [:quantifier, :one_or_more, '+', 15, 16]
|
85
49
|
end
|
data/spec/scanner/meta_spec.rb
CHANGED
@@ -1,38 +1,18 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe('Meta scanning') do
|
4
|
-
|
5
|
-
'abc
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
5 => [:meta, :alternation, '|', 11, 12],
|
12
|
-
},
|
4
|
+
include_examples 'scan', /abc??|def*+|ghi+/,
|
5
|
+
0 => [:literal, :literal, 'abc', 0, 3],
|
6
|
+
1 => [:quantifier, :zero_or_one_reluctant, '??', 3, 5],
|
7
|
+
2 => [:meta, :alternation, '|', 5, 6],
|
8
|
+
3 => [:literal, :literal, 'def', 6, 9],
|
9
|
+
4 => [:quantifier, :zero_or_more_possessive, '*+', 9, 11],
|
10
|
+
5 => [:meta, :alternation, '|', 11, 12]
|
13
11
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
},
|
21
|
-
}
|
22
|
-
|
23
|
-
tests.each_with_index do |(pattern, checks), count|
|
24
|
-
specify("scanner_meta_alternation_#{count}") do
|
25
|
-
tokens = RS.scan(pattern)
|
26
|
-
|
27
|
-
checks.each do |index, (type, token, text, ts, te)|
|
28
|
-
result = tokens.at(index)
|
29
|
-
|
30
|
-
expect(result[0]).to eq type
|
31
|
-
expect(result[1]).to eq token
|
32
|
-
expect(result[2]).to eq text
|
33
|
-
expect(result[3]).to eq ts
|
34
|
-
expect(result[4]).to eq te
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
12
|
+
include_examples 'scan', /(a\|b)|(c|d)\|(e[|]f)/,
|
13
|
+
2 => [:escape, :alternation, '\|', 2, 4],
|
14
|
+
5 => [:meta, :alternation, '|', 6, 7],
|
15
|
+
8 => [:meta, :alternation, '|', 9, 10],
|
16
|
+
11 => [:escape, :alternation, '\|', 12, 14],
|
17
|
+
15 => [:literal, :literal, '|', 17, 18]
|
38
18
|
end
|
@@ -1,307 +1,64 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe('Property scanning') do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
'Newline' => :newline,
|
10
|
-
|
11
|
-
'Any' => :any,
|
12
|
-
|
13
|
-
'Assigned' => :assigned,
|
14
|
-
|
15
|
-
'L' => :letter,
|
16
|
-
'Letter' => :letter,
|
17
|
-
|
18
|
-
'Lu' => :uppercase_letter,
|
19
|
-
'Uppercase_Letter' => :uppercase_letter,
|
20
|
-
|
21
|
-
'Ll' => :lowercase_letter,
|
22
|
-
'Lowercase_Letter' => :lowercase_letter,
|
23
|
-
|
24
|
-
'Lt' => :titlecase_letter,
|
25
|
-
'Titlecase_Letter' => :titlecase_letter,
|
26
|
-
|
27
|
-
'Lm' => :modifier_letter,
|
28
|
-
'Modifier_Letter' => :modifier_letter,
|
29
|
-
|
30
|
-
'Lo' => :other_letter,
|
31
|
-
'Other_Letter' => :other_letter,
|
32
|
-
|
33
|
-
'M' => :mark,
|
34
|
-
'Mark' => :mark,
|
35
|
-
|
36
|
-
'Mn' => :nonspacing_mark,
|
37
|
-
'Nonspacing_Mark' => :nonspacing_mark,
|
38
|
-
|
39
|
-
'Mc' => :spacing_mark,
|
40
|
-
'Spacing_Mark' => :spacing_mark,
|
41
|
-
|
42
|
-
'Me' => :enclosing_mark,
|
43
|
-
'Enclosing_Mark' => :enclosing_mark,
|
44
|
-
|
45
|
-
'N' => :number,
|
46
|
-
'Number' => :number,
|
47
|
-
|
48
|
-
'Nd' => :decimal_number,
|
49
|
-
'Decimal_Number' => :decimal_number,
|
50
|
-
|
51
|
-
'Nl' => :letter_number,
|
52
|
-
'Letter_Number' => :letter_number,
|
53
|
-
|
54
|
-
'No' => :other_number,
|
55
|
-
'Other_Number' => :other_number,
|
56
|
-
|
57
|
-
'P' => :punctuation,
|
58
|
-
'Punctuation' => :punctuation,
|
59
|
-
|
60
|
-
'Pc' => :connector_punctuation,
|
61
|
-
'Connector_Punctuation' => :connector_punctuation,
|
62
|
-
|
63
|
-
'Pd' => :dash_punctuation,
|
64
|
-
'Dash_Punctuation' => :dash_punctuation,
|
65
|
-
|
66
|
-
'Ps' => :open_punctuation,
|
67
|
-
'Open_Punctuation' => :open_punctuation,
|
68
|
-
|
69
|
-
'Pe' => :close_punctuation,
|
70
|
-
'Close_Punctuation' => :close_punctuation,
|
71
|
-
|
72
|
-
'Pi' => :initial_punctuation,
|
73
|
-
'Initial_Punctuation' => :initial_punctuation,
|
74
|
-
|
75
|
-
'Pf' => :final_punctuation,
|
76
|
-
'Final_Punctuation' => :final_punctuation,
|
77
|
-
|
78
|
-
'Po' => :other_punctuation,
|
79
|
-
'Other_Punctuation' => :other_punctuation,
|
80
|
-
|
81
|
-
'S' => :symbol,
|
82
|
-
'Symbol' => :symbol,
|
83
|
-
|
84
|
-
'Sm' => :math_symbol,
|
85
|
-
'Math_Symbol' => :math_symbol,
|
86
|
-
|
87
|
-
'Sc' => :currency_symbol,
|
88
|
-
'Currency_Symbol' => :currency_symbol,
|
89
|
-
|
90
|
-
'Sk' => :modifier_symbol,
|
91
|
-
'Modifier_Symbol' => :modifier_symbol,
|
92
|
-
|
93
|
-
'So' => :other_symbol,
|
94
|
-
'Other_Symbol' => :other_symbol,
|
95
|
-
|
96
|
-
'Z' => :separator,
|
97
|
-
'Separator' => :separator,
|
98
|
-
|
99
|
-
'Zs' => :space_separator,
|
100
|
-
'Space_Separator' => :space_separator,
|
101
|
-
|
102
|
-
'Zl' => :line_separator,
|
103
|
-
'Line_Separator' => :line_separator,
|
104
|
-
|
105
|
-
'Zp' => :paragraph_separator,
|
106
|
-
'Paragraph_Separator' => :paragraph_separator,
|
107
|
-
|
108
|
-
'C' => :other,
|
109
|
-
'Other' => :other,
|
110
|
-
|
111
|
-
'Cc' => :control,
|
112
|
-
'Control' => :control,
|
113
|
-
|
114
|
-
'Cf' => :format,
|
115
|
-
'Format' => :format,
|
116
|
-
|
117
|
-
'Cs' => :surrogate,
|
118
|
-
'Surrogate' => :surrogate,
|
119
|
-
|
120
|
-
'Co' => :private_use,
|
121
|
-
'Private_Use' => :private_use,
|
122
|
-
|
123
|
-
'Cn' => :unassigned,
|
124
|
-
'Unassigned' => :unassigned,
|
125
|
-
|
126
|
-
'Age=1.1' => :'age=1.1',
|
127
|
-
'Age=6.0' => :'age=6.0',
|
128
|
-
'Age=10.0' => :'age=10.0',
|
129
|
-
|
130
|
-
'ahex' => :ascii_hex_digit,
|
131
|
-
'ASCII_Hex_Digit' => :ascii_hex_digit,
|
132
|
-
|
133
|
-
'Alphabetic' => :alphabetic,
|
134
|
-
|
135
|
-
'Cased' => :cased,
|
136
|
-
|
137
|
-
'cwcf' => :changes_when_casefolded,
|
138
|
-
'Changes_When_Casefolded' => :changes_when_casefolded,
|
139
|
-
|
140
|
-
'cwcm' => :changes_when_casemapped,
|
141
|
-
'Changes_When_Casemapped' => :changes_when_casemapped,
|
142
|
-
|
143
|
-
'cwl' => :changes_when_lowercased,
|
144
|
-
'Changes_When_Lowercased' => :changes_when_lowercased,
|
145
|
-
|
146
|
-
'cwt' => :changes_when_titlecased,
|
147
|
-
'Changes_When_Titlecased' => :changes_when_titlecased,
|
148
|
-
|
149
|
-
'cwu' => :changes_when_uppercased,
|
150
|
-
'Changes_When_Uppercased' => :changes_when_uppercased,
|
151
|
-
|
152
|
-
'ci' => :case_ignorable,
|
153
|
-
'Case_Ignorable' => :case_ignorable,
|
154
|
-
|
155
|
-
'bidic' => :bidi_control,
|
156
|
-
'Bidi_Control' => :bidi_control,
|
157
|
-
|
158
|
-
'Dash' => :dash,
|
159
|
-
|
160
|
-
'dep' => :deprecated,
|
161
|
-
'Deprecated' => :deprecated,
|
162
|
-
|
163
|
-
'di' => :default_ignorable_code_point,
|
164
|
-
'Default_Ignorable_Code_Point' => :default_ignorable_code_point,
|
165
|
-
|
166
|
-
'dia' => :diacritic,
|
167
|
-
'Diacritic' => :diacritic,
|
168
|
-
|
169
|
-
'ext' => :extender,
|
170
|
-
'Extender' => :extender,
|
171
|
-
|
172
|
-
'grbase' => :grapheme_base,
|
173
|
-
'Grapheme_Base' => :grapheme_base,
|
174
|
-
|
175
|
-
'grext' => :grapheme_extend,
|
176
|
-
'Grapheme_Extend' => :grapheme_extend,
|
177
|
-
|
178
|
-
'grlink' => :grapheme_link,
|
179
|
-
'Grapheme_Link' => :grapheme_link,
|
180
|
-
|
181
|
-
'hex' => :hex_digit,
|
182
|
-
'Hex_Digit' => :hex_digit,
|
183
|
-
|
184
|
-
'Hyphen' => :hyphen,
|
185
|
-
|
186
|
-
'idc' => :id_continue,
|
187
|
-
'ID_Continue' => :id_continue,
|
188
|
-
|
189
|
-
'ideo' => :ideographic,
|
190
|
-
'Ideographic' => :ideographic,
|
191
|
-
|
192
|
-
'ids' => :id_start,
|
193
|
-
'ID_Start' => :id_start,
|
194
|
-
|
195
|
-
'idsb' => :ids_binary_operator,
|
196
|
-
'IDS_Binary_Operator' => :ids_binary_operator,
|
197
|
-
|
198
|
-
'idst' => :ids_trinary_operator,
|
199
|
-
'IDS_Trinary_Operator' => :ids_trinary_operator,
|
200
|
-
|
201
|
-
'joinc' => :join_control,
|
202
|
-
'Join_Control' => :join_control,
|
203
|
-
|
204
|
-
'loe' => :logical_order_exception,
|
205
|
-
'Logical_Order_Exception' => :logical_order_exception,
|
206
|
-
|
207
|
-
'Lowercase' => :lowercase,
|
208
|
-
|
209
|
-
'Math' => :math,
|
210
|
-
|
211
|
-
'nchar' => :noncharacter_code_point,
|
212
|
-
'Noncharacter_Code_Point' => :noncharacter_code_point,
|
213
|
-
|
214
|
-
'oalpha' => :other_alphabetic,
|
215
|
-
'Other_Alphabetic' => :other_alphabetic,
|
216
|
-
|
217
|
-
'odi' => :other_default_ignorable_code_point,
|
218
|
-
'Other_Default_Ignorable_Code_Point' => :other_default_ignorable_code_point,
|
219
|
-
|
220
|
-
'ogrext' => :other_grapheme_extend,
|
221
|
-
'Other_Grapheme_Extend' => :other_grapheme_extend,
|
222
|
-
|
223
|
-
'oidc' => :other_id_continue,
|
224
|
-
'Other_ID_Continue' => :other_id_continue,
|
225
|
-
|
226
|
-
'oids' => :other_id_start,
|
227
|
-
'Other_ID_Start' => :other_id_start,
|
228
|
-
|
229
|
-
'olower' => :other_lowercase,
|
230
|
-
'Other_Lowercase' => :other_lowercase,
|
231
|
-
|
232
|
-
'omath' => :other_math,
|
233
|
-
'Other_Math' => :other_math,
|
234
|
-
|
235
|
-
'oupper' => :other_uppercase,
|
236
|
-
'Other_Uppercase' => :other_uppercase,
|
4
|
+
RSpec.shared_examples 'scan property' do |text, token|
|
5
|
+
it("scans \\p{#{text}} as property #{token}") do
|
6
|
+
result = RS.scan("\\p{#{text}}")[0]
|
7
|
+
expect(result[0..1]).to eq [:property, token]
|
8
|
+
end
|
237
9
|
|
238
|
-
|
239
|
-
|
10
|
+
it("scans \\P{#{text}} as nonproperty #{token}") do
|
11
|
+
result = RS.scan("\\P{#{text}}")[0]
|
12
|
+
expect(result[0..1]).to eq [:nonproperty, token]
|
13
|
+
end
|
240
14
|
|
241
|
-
|
242
|
-
|
15
|
+
it("scans \\p{^#{text}} as nonproperty #{token}") do
|
16
|
+
result = RS.scan("\\p{^#{text}}")[0]
|
17
|
+
expect(result[0..1]).to eq [:nonproperty, token]
|
18
|
+
end
|
243
19
|
|
244
|
-
|
245
|
-
|
20
|
+
it("scans double-negated \\P{^#{text}} as property #{token}") do
|
21
|
+
result = RS.scan("\\P{^#{text}}")[0]
|
22
|
+
expect(result[0..1]).to eq [:property, token]
|
23
|
+
end
|
24
|
+
end
|
246
25
|
|
247
|
-
|
26
|
+
include_examples 'scan property', 'Alnum', :alnum
|
248
27
|
|
249
|
-
|
250
|
-
'Regional_Indicator' => :regional_indicator,
|
28
|
+
include_examples 'scan property', 'XPosixPunct', :xposixpunct
|
251
29
|
|
252
|
-
|
253
|
-
'Soft-Dotted' => :soft_dotted, # test dash spelling
|
30
|
+
include_examples 'scan property', 'Newline', :newline
|
254
31
|
|
255
|
-
|
32
|
+
include_examples 'scan property', 'Any', :any
|
256
33
|
|
257
|
-
|
258
|
-
'Terminal_Punctuation' => :terminal_punctuation,
|
34
|
+
include_examples 'scan property', 'Assigned', :assigned
|
259
35
|
|
260
|
-
|
261
|
-
|
36
|
+
include_examples 'scan property', 'Age=1.1', :'age=1.1'
|
37
|
+
include_examples 'scan property', 'Age=10.0', :'age=10.0'
|
262
38
|
|
263
|
-
|
39
|
+
include_examples 'scan property', 'ahex', :ascii_hex_digit
|
40
|
+
include_examples 'scan property', 'ASCII_Hex_Digit', :ascii_hex_digit # test underscore
|
264
41
|
|
265
|
-
|
266
|
-
|
42
|
+
include_examples 'scan property', 'sd', :soft_dotted
|
43
|
+
include_examples 'scan property', 'Soft-Dotted', :soft_dotted # test dash
|
267
44
|
|
268
|
-
|
269
|
-
|
45
|
+
include_examples 'scan property', 'Egyp', :egyptian_hieroglyphs
|
46
|
+
include_examples 'scan property', 'Egyptian Hieroglyphs', :egyptian_hieroglyphs # test whitespace
|
270
47
|
|
271
|
-
|
272
|
-
|
48
|
+
include_examples 'scan property', 'Linb', :linear_b
|
49
|
+
include_examples 'scan property', 'Linear-B', :linear_b # test dash
|
273
50
|
|
274
|
-
|
275
|
-
|
276
|
-
|
51
|
+
include_examples 'scan property', 'InArabic', :in_arabic # test block
|
52
|
+
include_examples 'scan property', 'in Arabic', :in_arabic # test block w. whitespace
|
53
|
+
include_examples 'scan property', 'In_Arabic', :in_arabic # test block w. underscore
|
277
54
|
|
278
|
-
|
279
|
-
|
280
|
-
tokens = RS.scan("a\\p{#{property}}c")
|
281
|
-
result = tokens.at(1)
|
282
|
-
expect(result[0]).to eq :property
|
283
|
-
expect(result[1]).to eq token
|
284
|
-
end
|
55
|
+
include_examples 'scan property', 'Yiii', :yi
|
56
|
+
include_examples 'scan property', 'Yi', :yi
|
285
57
|
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
expect(result[0]).to eq :nonproperty
|
290
|
-
expect(result[1]).to eq token
|
291
|
-
end
|
292
|
-
|
293
|
-
specify("scan_caret_nonproperty_#{token}_#{count}") do
|
294
|
-
tokens = RS.scan("a\\p{^#{property}}c")
|
295
|
-
result = tokens.at(1)
|
296
|
-
expect(result[0]).to eq :nonproperty
|
297
|
-
expect(result[1]).to eq token
|
298
|
-
end
|
58
|
+
include_examples 'scan property', 'Zinh', :inherited
|
59
|
+
include_examples 'scan property', 'Inherited', :inherited
|
60
|
+
include_examples 'scan property', 'Qaai', :inherited
|
299
61
|
|
300
|
-
|
301
|
-
|
302
|
-
result = tokens.at(1)
|
303
|
-
expect(result[0]).to eq :property
|
304
|
-
expect(result[1]).to eq token
|
305
|
-
end
|
306
|
-
end
|
62
|
+
include_examples 'scan property', 'Zzzz', :unknown
|
63
|
+
include_examples 'scan property', 'Unknown', :unknown
|
307
64
|
end
|