regexp_parser 2.1.1 → 2.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (129) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +19 -1
  3. data/LICENSE +1 -1
  4. data/README.md +17 -23
  5. data/Rakefile +10 -19
  6. data/lib/regexp_parser/expression/base.rb +123 -0
  7. data/lib/regexp_parser/expression/classes/anchor.rb +0 -2
  8. data/lib/regexp_parser/expression/classes/{backref.rb → backreference.rb} +0 -0
  9. data/lib/regexp_parser/expression/classes/{set → character_set}/intersection.rb +0 -0
  10. data/lib/regexp_parser/expression/classes/{set → character_set}/range.rb +0 -0
  11. data/lib/regexp_parser/expression/classes/{set.rb → character_set.rb} +0 -0
  12. data/lib/regexp_parser/expression/classes/{escape.rb → escape_sequence.rb} +13 -7
  13. data/lib/regexp_parser/expression/classes/free_space.rb +0 -2
  14. data/lib/regexp_parser/expression/classes/literal.rb +1 -5
  15. data/lib/regexp_parser/expression/classes/property.rb +0 -2
  16. data/lib/regexp_parser/expression/classes/root.rb +0 -1
  17. data/lib/regexp_parser/expression/classes/type.rb +0 -2
  18. data/lib/regexp_parser/expression/methods/strfregexp.rb +1 -1
  19. data/lib/regexp_parser/expression/quantifier.rb +1 -1
  20. data/lib/regexp_parser/expression/sequence.rb +0 -1
  21. data/lib/regexp_parser/expression/subexpression.rb +0 -1
  22. data/lib/regexp_parser/expression.rb +6 -130
  23. data/lib/regexp_parser/lexer.rb +7 -5
  24. data/lib/regexp_parser/scanner/properties/long.csv +604 -0
  25. data/lib/regexp_parser/scanner/properties/short.csv +242 -0
  26. data/lib/regexp_parser/scanner/scanner.rl +6 -4
  27. data/lib/regexp_parser/scanner.rb +126 -124
  28. data/lib/regexp_parser/syntax/any.rb +1 -3
  29. data/lib/regexp_parser/syntax/base.rb +12 -14
  30. data/lib/regexp_parser/syntax/token/anchor.rb +15 -0
  31. data/lib/regexp_parser/syntax/{tokens → token}/assertion.rb +2 -2
  32. data/lib/regexp_parser/syntax/token/backreference.rb +30 -0
  33. data/lib/regexp_parser/syntax/{tokens → token}/character_set.rb +2 -2
  34. data/lib/regexp_parser/syntax/{tokens → token}/character_type.rb +3 -3
  35. data/lib/regexp_parser/syntax/{tokens → token}/conditional.rb +3 -3
  36. data/lib/regexp_parser/syntax/token/escape.rb +31 -0
  37. data/lib/regexp_parser/syntax/{tokens → token}/group.rb +7 -7
  38. data/lib/regexp_parser/syntax/{tokens → token}/keep.rb +1 -1
  39. data/lib/regexp_parser/syntax/{tokens → token}/meta.rb +2 -2
  40. data/lib/regexp_parser/syntax/{tokens → token}/posix_class.rb +3 -3
  41. data/lib/regexp_parser/syntax/token/quantifier.rb +35 -0
  42. data/lib/regexp_parser/syntax/token/unicode_property.rb +696 -0
  43. data/lib/regexp_parser/syntax/token.rb +45 -0
  44. data/lib/regexp_parser/syntax/versions/1.8.6.rb +2 -2
  45. data/lib/regexp_parser/syntax/versions/1.9.1.rb +1 -1
  46. data/lib/regexp_parser/syntax/versions/3.1.0.rb +10 -0
  47. data/lib/regexp_parser/syntax.rb +1 -1
  48. data/lib/regexp_parser/token.rb +9 -20
  49. data/lib/regexp_parser/version.rb +1 -1
  50. data/lib/regexp_parser.rb +0 -2
  51. data/regexp_parser.gemspec +20 -22
  52. metadata +32 -164
  53. data/lib/regexp_parser/scanner/properties/long.yml +0 -594
  54. data/lib/regexp_parser/scanner/properties/short.yml +0 -237
  55. data/lib/regexp_parser/syntax/tokens/anchor.rb +0 -15
  56. data/lib/regexp_parser/syntax/tokens/backref.rb +0 -24
  57. data/lib/regexp_parser/syntax/tokens/escape.rb +0 -30
  58. data/lib/regexp_parser/syntax/tokens/quantifier.rb +0 -35
  59. data/lib/regexp_parser/syntax/tokens/unicode_property.rb +0 -675
  60. data/lib/regexp_parser/syntax/tokens.rb +0 -45
  61. data/spec/expression/base_spec.rb +0 -104
  62. data/spec/expression/clone_spec.rb +0 -152
  63. data/spec/expression/conditional_spec.rb +0 -89
  64. data/spec/expression/free_space_spec.rb +0 -27
  65. data/spec/expression/methods/match_length_spec.rb +0 -161
  66. data/spec/expression/methods/match_spec.rb +0 -25
  67. data/spec/expression/methods/strfregexp_spec.rb +0 -224
  68. data/spec/expression/methods/tests_spec.rb +0 -99
  69. data/spec/expression/methods/traverse_spec.rb +0 -161
  70. data/spec/expression/options_spec.rb +0 -128
  71. data/spec/expression/subexpression_spec.rb +0 -50
  72. data/spec/expression/to_h_spec.rb +0 -26
  73. data/spec/expression/to_s_spec.rb +0 -108
  74. data/spec/lexer/all_spec.rb +0 -22
  75. data/spec/lexer/conditionals_spec.rb +0 -53
  76. data/spec/lexer/delimiters_spec.rb +0 -68
  77. data/spec/lexer/escapes_spec.rb +0 -14
  78. data/spec/lexer/keep_spec.rb +0 -10
  79. data/spec/lexer/literals_spec.rb +0 -64
  80. data/spec/lexer/nesting_spec.rb +0 -99
  81. data/spec/lexer/refcalls_spec.rb +0 -60
  82. data/spec/parser/all_spec.rb +0 -43
  83. data/spec/parser/alternation_spec.rb +0 -88
  84. data/spec/parser/anchors_spec.rb +0 -17
  85. data/spec/parser/conditionals_spec.rb +0 -179
  86. data/spec/parser/errors_spec.rb +0 -30
  87. data/spec/parser/escapes_spec.rb +0 -121
  88. data/spec/parser/free_space_spec.rb +0 -130
  89. data/spec/parser/groups_spec.rb +0 -108
  90. data/spec/parser/keep_spec.rb +0 -6
  91. data/spec/parser/options_spec.rb +0 -28
  92. data/spec/parser/posix_classes_spec.rb +0 -8
  93. data/spec/parser/properties_spec.rb +0 -115
  94. data/spec/parser/quantifiers_spec.rb +0 -68
  95. data/spec/parser/refcalls_spec.rb +0 -117
  96. data/spec/parser/set/intersections_spec.rb +0 -127
  97. data/spec/parser/set/ranges_spec.rb +0 -111
  98. data/spec/parser/sets_spec.rb +0 -178
  99. data/spec/parser/types_spec.rb +0 -18
  100. data/spec/scanner/all_spec.rb +0 -18
  101. data/spec/scanner/anchors_spec.rb +0 -21
  102. data/spec/scanner/conditionals_spec.rb +0 -128
  103. data/spec/scanner/delimiters_spec.rb +0 -52
  104. data/spec/scanner/errors_spec.rb +0 -67
  105. data/spec/scanner/escapes_spec.rb +0 -64
  106. data/spec/scanner/free_space_spec.rb +0 -165
  107. data/spec/scanner/groups_spec.rb +0 -61
  108. data/spec/scanner/keep_spec.rb +0 -10
  109. data/spec/scanner/literals_spec.rb +0 -39
  110. data/spec/scanner/meta_spec.rb +0 -18
  111. data/spec/scanner/options_spec.rb +0 -36
  112. data/spec/scanner/properties_spec.rb +0 -64
  113. data/spec/scanner/quantifiers_spec.rb +0 -25
  114. data/spec/scanner/refcalls_spec.rb +0 -55
  115. data/spec/scanner/sets_spec.rb +0 -151
  116. data/spec/scanner/types_spec.rb +0 -14
  117. data/spec/spec_helper.rb +0 -16
  118. data/spec/support/runner.rb +0 -42
  119. data/spec/support/shared_examples.rb +0 -77
  120. data/spec/support/warning_extractor.rb +0 -60
  121. data/spec/syntax/syntax_spec.rb +0 -48
  122. data/spec/syntax/syntax_token_map_spec.rb +0 -23
  123. data/spec/syntax/versions/1.8.6_spec.rb +0 -17
  124. data/spec/syntax/versions/1.9.1_spec.rb +0 -10
  125. data/spec/syntax/versions/1.9.3_spec.rb +0 -9
  126. data/spec/syntax/versions/2.0.0_spec.rb +0 -13
  127. data/spec/syntax/versions/2.2.0_spec.rb +0 -9
  128. data/spec/syntax/versions/aliases_spec.rb +0 -37
  129. data/spec/token/token_spec.rb +0 -85
@@ -1,165 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('FreeSpace scanning') do
4
- describe('scan free space tokens') do
5
- let(:tokens) { RS.scan(/
6
- a
7
- b ? c *
8
- d {2,3}
9
- e + | f +
10
- /x) }
11
-
12
- 0.upto(24).select(&:even?).each do |i|
13
- it "scans #{i} as free space" do
14
- expect(tokens[i][0]).to eq :free_space
15
- expect(tokens[i][1]).to eq :whitespace
16
- end
17
- end
18
- 0.upto(24).reject(&:even?).each do |i|
19
- it "does not scan #{i} as free space" do
20
- expect(tokens[i][0]).not_to eq :free_space
21
- expect(tokens[i][1]).not_to eq :whitespace
22
- end
23
- end
24
-
25
- it 'sets the correct text' do
26
- [0, 2, 10, 14].each { |i| expect(tokens[i][2]).to eq "\n " }
27
- [4, 6, 8, 12].each { |i| expect(tokens[i][2]).to eq ' ' }
28
- end
29
- end
30
-
31
- describe('scan free space comments') do
32
- include_examples 'scan', /
33
- a + # A + comment
34
- b ? # B ? comment
35
- c {2,3} # C {2,3} comment
36
- d + | e + # D|E comment
37
- /x,
38
- 5 => [:free_space, :comment, "# A + comment\n", 11, 25],
39
- 11 => [:free_space, :comment, "# B ? comment\n", 37, 51],
40
- 17 => [:free_space, :comment, "# C {2,3} comment\n", 66, 84],
41
- 29 => [:free_space, :comment, "# D|E comment\n", 100, 114]
42
-
43
- # single line / no trailing newline (c.f. issue #66)
44
- include_examples 'scan', /a # b/x,
45
- 0 => [:literal, :literal, 'a', 0, 1],
46
- 1 => [:free_space, :whitespace, ' ', 1, 2],
47
- 2 => [:free_space, :comment, "# b", 2, 5]
48
-
49
- # without spaces (c.f. issue #66)
50
- include_examples 'scan', /a#b/x,
51
- 0 => [:literal, :literal, 'a', 0, 1],
52
- 1 => [:free_space, :comment, "#b", 1, 3]
53
- end
54
-
55
- describe('scan free space inlined') do
56
- include_examples 'scan', /a b(?x:c d e)f g/,
57
- 0 => [:literal, :literal, 'a b', 0, 3],
58
- 1 => [:group, :options, '(?x:', 3, 7],
59
- 2 => [:literal, :literal, 'c', 7, 8],
60
- 3 => [:free_space, :whitespace, ' ', 8, 9],
61
- 4 => [:literal, :literal, 'd', 9, 10],
62
- 5 => [:free_space, :whitespace, ' ', 10, 11],
63
- 6 => [:literal, :literal, 'e', 11, 12],
64
- 7 => [:group, :close, ')', 12, 13],
65
- 8 => [:literal, :literal, 'f g', 13, 16]
66
- end
67
-
68
- describe('scan free space nested') do
69
- include_examples 'scan', /a b(?x:c d(?-x:e f)g h)i j/,
70
- 0 => [:literal, :literal, 'a b', 0, 3],
71
- 1 => [:group, :options, '(?x:', 3, 7],
72
- 2 => [:literal, :literal, 'c', 7, 8],
73
- 3 => [:free_space, :whitespace, ' ', 8, 9],
74
- 4 => [:literal, :literal, 'd', 9, 10],
75
- 5 => [:group, :options, '(?-x:', 10, 15],
76
- 6 => [:literal, :literal, 'e f', 15, 18],
77
- 7 => [:group, :close, ')', 18, 19],
78
- 8 => [:literal, :literal, 'g', 19, 20],
79
- 9 => [:free_space, :whitespace, ' ', 20, 21],
80
- 10 => [:literal, :literal, 'h', 21, 22],
81
- 11 => [:group, :close, ')', 22, 23],
82
- 12 => [:literal, :literal, 'i j', 23, 26]
83
- end
84
-
85
- describe('scan free space nested groups') do
86
- include_examples 'scan', /(a (b(?x: (c d) (?-x:(e f) )g) h)i j)/,
87
- 0 => [:group, :capture, '(', 0, 1],
88
- 1 => [:literal, :literal, 'a ', 1, 3],
89
- 2 => [:group, :capture, '(', 3, 4],
90
- 3 => [:literal, :literal, 'b', 4, 5],
91
- 4 => [:group, :options, '(?x:', 5, 9],
92
- 5 => [:free_space, :whitespace, ' ', 9, 10],
93
- 6 => [:group, :capture, '(', 10, 11],
94
- 7 => [:literal, :literal, 'c', 11, 12],
95
- 8 => [:free_space, :whitespace, ' ', 12, 13],
96
- 9 => [:literal, :literal, 'd', 13, 14],
97
- 10 => [:group, :close, ')', 14, 15],
98
- 11 => [:free_space, :whitespace, ' ', 15, 16],
99
- 12 => [:group, :options, '(?-x:', 16, 21],
100
- 13 => [:group, :capture, '(', 21, 22],
101
- 14 => [:literal, :literal, 'e f', 22, 25],
102
- 15 => [:group, :close, ')', 25, 26],
103
- 16 => [:literal, :literal, ' ', 26, 27],
104
- 17 => [:group, :close, ')', 27, 28],
105
- 18 => [:literal, :literal, 'g', 28, 29],
106
- 19 => [:group, :close, ')', 29, 30],
107
- 20 => [:literal, :literal, ' h', 30, 32],
108
- 21 => [:group, :close, ')', 32, 33],
109
- 22 => [:literal, :literal, 'i j', 33, 36],
110
- 23 => [:group, :close, ')', 36, 37]
111
- end
112
-
113
- describe('scan free space switch groups') do
114
- include_examples 'scan', /(a (b((?x) (c d) ((?-x)(e f) )g) h)i j)/,
115
- 0 => [:group, :capture, '(', 0, 1],
116
- 1 => [:literal, :literal, 'a ', 1, 3],
117
- 2 => [:group, :capture, '(', 3, 4],
118
- 3 => [:literal, :literal, 'b', 4, 5],
119
- 4 => [:group, :capture, '(', 5, 6],
120
- 5 => [:group, :options_switch, '(?x', 6, 9],
121
- 6 => [:group, :close, ')', 9, 10],
122
- 7 => [:free_space, :whitespace, ' ', 10, 11],
123
- 8 => [:group, :capture, '(', 11, 12],
124
- 9 => [:literal, :literal, 'c', 12, 13],
125
- 10 => [:free_space, :whitespace, ' ', 13, 14],
126
- 11 => [:literal, :literal, 'd', 14, 15],
127
- 12 => [:group, :close, ')', 15, 16],
128
- 13 => [:free_space, :whitespace, ' ', 16, 17],
129
- 14 => [:group, :capture, '(', 17, 18],
130
- 15 => [:group, :options_switch, '(?-x', 18, 22],
131
- 16 => [:group, :close, ')', 22, 23],
132
- 17 => [:group, :capture, '(', 23, 24],
133
- 18 => [:literal, :literal, 'e f', 24, 27],
134
- 19 => [:group, :close, ')', 27, 28],
135
- 20 => [:literal, :literal, ' ', 28, 29],
136
- 21 => [:group, :close, ')', 29, 30],
137
- 22 => [:literal, :literal, 'g', 30, 31],
138
- 23 => [:group, :close, ')', 31, 32],
139
- 24 => [:literal, :literal, ' h', 32, 34],
140
- 25 => [:group, :close, ')', 34, 35],
141
- 26 => [:literal, :literal, 'i j', 35, 38],
142
- 27 => [:group, :close, ')', 38, 39]
143
- end
144
-
145
- describe('scanning `#` in regular (non-x mode)') do # c.f. issue 70
146
- include_examples 'scan', /a#bcd/,
147
- 0 => [:literal, :literal, 'a#bcd', 0, 5]
148
- include_examples 'scan', /a # bcd/,
149
- 0 => [:literal, :literal, 'a # bcd', 0, 7]
150
-
151
- include_examples 'scan', /a#\d/,
152
- 0 => [:literal, :literal, 'a#', 0, 2],
153
- 1 => [:type, :digit, '\d', 2, 4]
154
- include_examples 'scan', /a # \d/,
155
- 0 => [:literal, :literal, 'a # ', 0, 4],
156
- 1 => [:type, :digit, '\d', 4, 6]
157
-
158
- include_examples 'scan', /a#()/,
159
- 0 => [:literal, :literal, 'a#', 0, 2],
160
- 1 => [:group, :capture, '(', 2, 3]
161
- include_examples 'scan', /a # ()/,
162
- 0 => [:literal, :literal, 'a # ', 0, 4],
163
- 1 => [:group, :capture, '(', 4, 5]
164
- end
165
- end
@@ -1,61 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Group scanning') do
4
- # Group types
5
- include_examples 'scan', '(?>abc)', 0 => [:group, :atomic, '(?>', 0, 3]
6
- include_examples 'scan', '(abc)', 0 => [:group, :capture, '(', 0, 1]
7
-
8
- # Named groups
9
- # only names that start with a hyphen or digit (ascii or other) are invalid
10
- include_examples 'scan', '(?<name>abc)', 0 => [:group, :named_ab, '(?<name>', 0, 8]
11
- include_examples 'scan', "(?'name'abc)", 0 => [:group, :named_sq, "(?'name'", 0, 8]
12
- include_examples 'scan', '(?<name_1>abc)', 0 => [:group, :named_ab, '(?<name_1>', 0,10]
13
- include_examples 'scan', "(?'name_1'abc)", 0 => [:group, :named_sq, "(?'name_1'", 0,10]
14
- include_examples 'scan', '(?<name-1>abc)', 0 => [:group, :named_ab, '(?<name-1>', 0,10]
15
- include_examples 'scan', "(?'name-1'abc)", 0 => [:group, :named_sq, "(?'name-1'", 0,10]
16
- include_examples 'scan', "(?<name'1>abc)", 0 => [:group, :named_ab, "(?<name'1>", 0,10]
17
- include_examples 'scan', "(?'name>1'abc)", 0 => [:group, :named_sq, "(?'name>1'", 0,10]
18
- include_examples 'scan', '(?<üüuuüü>abc)', 0 => [:group, :named_ab, '(?<üüuuüü>', 0,10]
19
- include_examples 'scan', "(?'üüuuüü'abc)", 0 => [:group, :named_sq, "(?'üüuuüü'", 0,10]
20
- include_examples 'scan', "(?<😋1234😋>abc)", 0 => [:group, :named_ab, "(?<😋1234😋>", 0,10]
21
- include_examples 'scan', "(?'😋1234😋'abc)", 0 => [:group, :named_sq, "(?'😋1234😋'", 0,10]
22
-
23
- include_examples 'scan', '(?:abc)', 0 => [:group, :passive, '(?:', 0, 3]
24
- include_examples 'scan', '(?:)', 0 => [:group, :passive, '(?:', 0, 3]
25
- include_examples 'scan', '(?::)', 0 => [:group, :passive, '(?:', 0, 3]
26
-
27
- # Comments
28
- include_examples 'scan', '(?#abc)', 0 => [:group, :comment, '(?#abc)', 0, 7]
29
- include_examples 'scan', '(?#)', 0 => [:group, :comment, '(?#)', 0, 4]
30
-
31
- # Assertions
32
- include_examples 'scan', '(?=abc)', 0 => [:assertion, :lookahead, '(?=', 0, 3]
33
- include_examples 'scan', '(?!abc)', 0 => [:assertion, :nlookahead, '(?!', 0, 3]
34
- include_examples 'scan', '(?<=abc)', 0 => [:assertion, :lookbehind, '(?<=', 0, 4]
35
- include_examples 'scan', '(?<!abc)', 0 => [:assertion, :nlookbehind, '(?<!', 0, 4]
36
-
37
- # Options
38
- include_examples 'scan', '(?-mix:abc)', 0 => [:group, :options, '(?-mix:', 0, 7]
39
- include_examples 'scan', '(?m-ix:abc)', 0 => [:group, :options, '(?m-ix:', 0, 7]
40
- include_examples 'scan', '(?mi-x:abc)', 0 => [:group, :options, '(?mi-x:', 0, 7]
41
- include_examples 'scan', '(?mix:abc)', 0 => [:group, :options, '(?mix:', 0, 6]
42
- include_examples 'scan', '(?m:)', 0 => [:group, :options, '(?m:', 0, 4]
43
- include_examples 'scan', '(?i:)', 0 => [:group, :options, '(?i:', 0, 4]
44
- include_examples 'scan', '(?x:)', 0 => [:group, :options, '(?x:', 0, 4]
45
- include_examples 'scan', '(?mix)', 0 => [:group, :options_switch, '(?mix', 0, 5]
46
- include_examples 'scan', '(?d-mix:abc)', 0 => [:group, :options, '(?d-mix:', 0, 8]
47
- include_examples 'scan', '(?a-mix:abc)', 0 => [:group, :options, '(?a-mix:', 0, 8]
48
- include_examples 'scan', '(?u-mix:abc)', 0 => [:group, :options, '(?u-mix:', 0, 8]
49
- include_examples 'scan', '(?da-m:abc)', 0 => [:group, :options, '(?da-m:', 0, 7]
50
- include_examples 'scan', '(?du-x:abc)', 0 => [:group, :options, '(?du-x:', 0, 7]
51
- include_examples 'scan', '(?dau-i:abc)', 0 => [:group, :options, '(?dau-i:', 0, 8]
52
- include_examples 'scan', '(?dau:abc)', 0 => [:group, :options, '(?dau:', 0, 6]
53
- include_examples 'scan', '(?d:)', 0 => [:group, :options, '(?d:', 0, 4]
54
- include_examples 'scan', '(?a:)', 0 => [:group, :options, '(?a:', 0, 4]
55
- include_examples 'scan', '(?u:)', 0 => [:group, :options, '(?u:', 0, 4]
56
- include_examples 'scan', '(?dau)', 0 => [:group, :options_switch, '(?dau', 0, 5]
57
-
58
- if ruby_version_at_least('2.4.1')
59
- include_examples 'scan', '(?~abc)', 0 => [:group, :absence, '(?~', 0, 3]
60
- end
61
- end
@@ -1,10 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Keep scanning') do
4
- include_examples 'scan', /ab\Kcd/,
5
- 1 => [:keep, :mark, '\K', 2, 4]
6
-
7
- include_examples 'scan', /(a\Kb)|(c\\\Kd)ef/,
8
- 2 => [:keep, :mark, '\K', 2, 4],
9
- 9 => [:keep, :mark, '\K', 11, 13]
10
- end
@@ -1,39 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('UTF8 scanning') do
4
- # ascii, single byte characters
5
- include_examples 'scan', 'a',
6
- 0 => [:literal, :literal, 'a', 0, 1]
7
-
8
- include_examples 'scan', 'ab+',
9
- 0 => [:literal, :literal, 'ab', 0, 2],
10
- 1 => [:quantifier, :one_or_more, '+', 2, 3]
11
-
12
- # 2 byte wide characters
13
- include_examples 'scan', 'äöü',
14
- 0 => [:literal, :literal, 'äöü', 0, 3]
15
-
16
- # 3 byte wide characters, Japanese
17
- include_examples 'scan', 'ab?れます+cd',
18
- 0 => [:literal, :literal, 'ab', 0, 2],
19
- 1 => [:quantifier, :zero_or_one, '?', 2, 3],
20
- 2 => [:literal, :literal, 'れます', 3, 6],
21
- 3 => [:quantifier, :one_or_more, '+', 6, 7],
22
- 4 => [:literal, :literal, 'cd', 7, 9]
23
-
24
- # 4 byte wide characters, Osmanya
25
- include_examples 'scan', '𐒀𐒁?𐒂ab+𐒃',
26
- 0 => [:literal, :literal, '𐒀𐒁', 0, 2],
27
- 1 => [:quantifier, :zero_or_one, '?', 2, 3],
28
- 2 => [:literal, :literal, '𐒂ab', 3, 6],
29
- 3 => [:quantifier, :one_or_more, '+', 6, 7],
30
- 4 => [:literal, :literal, '𐒃', 7, 8]
31
-
32
- include_examples 'scan', 'mu𝄞?si*𝄫c+',
33
- 0 => [:literal, :literal, 'mu𝄞', 0, 3],
34
- 1 => [:quantifier, :zero_or_one, '?', 3, 4],
35
- 2 => [:literal, :literal, 'si', 4, 6],
36
- 3 => [:quantifier, :zero_or_more, '*', 6, 7],
37
- 4 => [:literal, :literal, '𝄫c', 7, 9],
38
- 5 => [:quantifier, :one_or_more, '+', 9, 10]
39
- end
@@ -1,18 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Meta scanning') do
4
- include_examples 'scan', /abc??|def*+|ghi+/,
5
- 0 => [:literal, :literal, 'abc', 0, 3],
6
- 1 => [:quantifier, :zero_or_one_reluctant, '??', 3, 5],
7
- 2 => [:meta, :alternation, '|', 5, 6],
8
- 3 => [:literal, :literal, 'def', 6, 9],
9
- 4 => [:quantifier, :zero_or_more_possessive, '*+', 9, 11],
10
- 5 => [:meta, :alternation, '|', 11, 12]
11
-
12
- include_examples 'scan', /(a\|b)|(c|d)\|(e[|]f)/,
13
- 2 => [:escape, :alternation, '\|', 2, 4],
14
- 5 => [:meta, :alternation, '|', 6, 7],
15
- 8 => [:meta, :alternation, '|', 9, 10],
16
- 11 => [:escape, :alternation, '\|', 12, 14],
17
- 15 => [:literal, :literal, '|', 17, 18]
18
- end
@@ -1,36 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('passing options to scan') do
4
- def expect_type_tokens(tokens, type_tokens)
5
- expect(tokens.map { |type, token, *| [type, token] }).to eq(type_tokens)
6
- end
7
-
8
- it 'raises if if scanning from a Regexp and options are passed' do
9
- expect { RS.scan(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
10
- ArgumentError,
11
- 'options cannot be supplied unless scanning a String'
12
- )
13
- end
14
-
15
- it 'sets free_spacing based on options if scanning from a String' do
16
- expect_type_tokens(
17
- RS.scan('a+#c', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED),
18
- [
19
- %i[literal literal],
20
- %i[quantifier one_or_more],
21
- %i[free_space comment]
22
- ]
23
- )
24
- end
25
-
26
- it 'does not set free_spacing if scanning from a String and passing no options' do
27
- expect_type_tokens(
28
- RS.scan('a+#c'),
29
- [
30
- %i[literal literal],
31
- %i[quantifier one_or_more],
32
- %i[literal literal]
33
- ]
34
- )
35
- end
36
- end
@@ -1,64 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Property scanning') do
4
- RSpec.shared_examples 'scan property' do |text, token|
5
- it("scans \\p{#{text}} as property #{token}") do
6
- result = RS.scan("\\p{#{text}}")[0]
7
- expect(result[0..1]).to eq [:property, token]
8
- end
9
-
10
- it("scans \\P{#{text}} as nonproperty #{token}") do
11
- result = RS.scan("\\P{#{text}}")[0]
12
- expect(result[0..1]).to eq [:nonproperty, token]
13
- end
14
-
15
- it("scans \\p{^#{text}} as nonproperty #{token}") do
16
- result = RS.scan("\\p{^#{text}}")[0]
17
- expect(result[0..1]).to eq [:nonproperty, token]
18
- end
19
-
20
- it("scans double-negated \\P{^#{text}} as property #{token}") do
21
- result = RS.scan("\\P{^#{text}}")[0]
22
- expect(result[0..1]).to eq [:property, token]
23
- end
24
- end
25
-
26
- include_examples 'scan property', 'Alnum', :alnum
27
-
28
- include_examples 'scan property', 'XPosixPunct', :xposixpunct
29
-
30
- include_examples 'scan property', 'Newline', :newline
31
-
32
- include_examples 'scan property', 'Any', :any
33
-
34
- include_examples 'scan property', 'Assigned', :assigned
35
-
36
- include_examples 'scan property', 'Age=1.1', :'age=1.1'
37
- include_examples 'scan property', 'Age=10.0', :'age=10.0'
38
-
39
- include_examples 'scan property', 'ahex', :ascii_hex_digit
40
- include_examples 'scan property', 'ASCII_Hex_Digit', :ascii_hex_digit # test underscore
41
-
42
- include_examples 'scan property', 'sd', :soft_dotted
43
- include_examples 'scan property', 'Soft-Dotted', :soft_dotted # test dash
44
-
45
- include_examples 'scan property', 'Egyp', :egyptian_hieroglyphs
46
- include_examples 'scan property', 'Egyptian Hieroglyphs', :egyptian_hieroglyphs # test whitespace
47
-
48
- include_examples 'scan property', 'Linb', :linear_b
49
- include_examples 'scan property', 'Linear-B', :linear_b # test dash
50
-
51
- include_examples 'scan property', 'InArabic', :in_arabic # test block
52
- include_examples 'scan property', 'in Arabic', :in_arabic # test block w. whitespace
53
- include_examples 'scan property', 'In_Arabic', :in_arabic # test block w. underscore
54
-
55
- include_examples 'scan property', 'Yiii', :yi
56
- include_examples 'scan property', 'Yi', :yi
57
-
58
- include_examples 'scan property', 'Zinh', :inherited
59
- include_examples 'scan property', 'Inherited', :inherited
60
- include_examples 'scan property', 'Qaai', :inherited
61
-
62
- include_examples 'scan property', 'Zzzz', :unknown
63
- include_examples 'scan property', 'Unknown', :unknown
64
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Quantifier scanning') do
4
- include_examples 'scan', 'a?', 1 => [:quantifier, :zero_or_one, '?', 1, 2]
5
- include_examples 'scan', 'a??', 1 => [:quantifier, :zero_or_one_reluctant, '??', 1, 3]
6
- include_examples 'scan', 'a?+', 1 => [:quantifier, :zero_or_one_possessive, '?+', 1, 3]
7
-
8
- include_examples 'scan', 'a*', 1 => [:quantifier, :zero_or_more, '*', 1, 2]
9
- include_examples 'scan', 'a*?', 1 => [:quantifier, :zero_or_more_reluctant, '*?', 1, 3]
10
- include_examples 'scan', 'a*+', 1 => [:quantifier, :zero_or_more_possessive, '*+', 1, 3]
11
-
12
- include_examples 'scan', 'a+', 1 => [:quantifier, :one_or_more, '+', 1, 2]
13
- include_examples 'scan', 'a+?', 1 => [:quantifier, :one_or_more_reluctant, '+?', 1, 3]
14
- include_examples 'scan', 'a++', 1 => [:quantifier, :one_or_more_possessive, '++', 1, 3]
15
-
16
- include_examples 'scan', 'a{2}', 1 => [:quantifier, :interval, '{2}', 1, 4]
17
- include_examples 'scan', 'a{2,}', 1 => [:quantifier, :interval, '{2,}', 1, 5]
18
- include_examples 'scan', 'a{,2}', 1 => [:quantifier, :interval, '{,2}', 1, 5]
19
- include_examples 'scan', 'a{2,4}', 1 => [:quantifier, :interval, '{2,4}', 1, 6]
20
-
21
- # special case: chained quantifiers
22
- include_examples 'scan', 'a+{2}{3}', 1 => [:quantifier, :one_or_more, '+', 1, 2]
23
- include_examples 'scan', 'a+{2}{3}', 2 => [:quantifier, :interval, '{2}', 2, 5]
24
- include_examples 'scan', 'a+{2}{3}', 3 => [:quantifier, :interval, '{3}', 5, 8]
25
- end
@@ -1,55 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('RefCall scanning') do
4
- # Traditional numerical group back-reference
5
- include_examples 'scan', '(abc)\1' , 3 => [:backref, :number, '\1', 5, 7]
6
-
7
- # Group back-references, named, numbered, and relative
8
- #
9
- # NOTE: only \g supports forward-looking references using '+', e.g. \g<+1>
10
- # refers to the next group, but \k<+1> refers to a group named '+1'.
11
- # Inversely, only \k supports addition or substraction of a recursion level.
12
- # E.g. \k<x+0> refers to a group named 'x' at the current recursion level,
13
- # but \g<x+0> refers to a a group named 'x+0'.
14
- #
15
- include_examples 'scan', '(?<X>abc)\k<X>', 3 => [:backref, :name_ref_ab, '\k<X>', 9, 14]
16
- include_examples 'scan', "(?<X>abc)\\k'X'", 3 => [:backref, :name_ref_sq, "\\k'X'", 9, 14]
17
-
18
- include_examples 'scan', '(?<+1>abc)\k<+1>', 3 => [:backref, :name_ref_ab, '\k<+1>', 10, 16]
19
- include_examples 'scan', "(?<+1>abc)\\k'+1'", 3 => [:backref, :name_ref_sq, "\\k'+1'", 10, 16]
20
-
21
- include_examples 'scan', '(abc)\k<1>', 3 => [:backref, :number_ref_ab, '\k<1>', 5, 10]
22
- include_examples 'scan', "(abc)\\k'1'", 3 => [:backref, :number_ref_sq, "\\k'1'", 5, 10]
23
-
24
- include_examples 'scan', '(abc)\k<-1>', 3 => [:backref, :number_rel_ref_ab, '\k<-1>', 5, 11]
25
- include_examples 'scan', "(abc)\\k'-1'", 3 => [:backref, :number_rel_ref_sq, "\\k'-1'", 5, 11]
26
-
27
- # Sub-expression invocation, named, numbered, and relative
28
- include_examples 'scan', '(?<X>abc)\g<X>', 3 => [:backref, :name_call_ab, '\g<X>', 9, 14]
29
- include_examples 'scan', "(?<X>abc)\\g'X'", 3 => [:backref, :name_call_sq, "\\g'X'", 9, 14]
30
-
31
- include_examples 'scan', '(?<X>abc)\g<X-1>', 3 => [:backref, :name_call_ab, '\g<X-1>', 9, 16]
32
- include_examples 'scan', "(?<X>abc)\\g'X-1'", 3 => [:backref, :name_call_sq, "\\g'X-1'", 9, 16]
33
-
34
- include_examples 'scan', '(abc)\g<1>', 3 => [:backref, :number_call_ab, '\g<1>', 5, 10]
35
- include_examples 'scan', "(abc)\\g'1'", 3 => [:backref, :number_call_sq, "\\g'1'", 5, 10]
36
-
37
- include_examples 'scan', 'a(b|\g<0>)', 4 => [:backref, :number_call_ab, '\g<0>', 4, 9]
38
- include_examples 'scan', "a(b|\\g'0')", 4 => [:backref, :number_call_sq, "\\g'0'", 4, 9]
39
-
40
- include_examples 'scan', '(abc)\g<-1>', 3 => [:backref, :number_rel_call_ab, '\g<-1>', 5, 11]
41
- include_examples 'scan', "(abc)\\g'-1'", 3 => [:backref, :number_rel_call_sq, "\\g'-1'", 5, 11]
42
-
43
- include_examples 'scan', '\g<+1>(abc)', 0 => [:backref, :number_rel_call_ab, '\g<+1>', 0, 6]
44
- include_examples 'scan', "\\g'+1'(abc)", 0 => [:backref, :number_rel_call_sq, "\\g'+1'", 0, 6]
45
-
46
- # Group back-references, with recursion level
47
- include_examples 'scan', '(?<X>abc)\k<X-0>', 3 => [:backref, :name_recursion_ref_ab, '\k<X-0>', 9, 16]
48
- include_examples 'scan', "(?<X>abc)\\k'X-0'", 3 => [:backref, :name_recursion_ref_sq, "\\k'X-0'", 9, 16]
49
-
50
- include_examples 'scan', '(abc)\k<1-0>', 3 => [:backref, :number_recursion_ref_ab, '\k<1-0>', 5, 12]
51
- include_examples 'scan', "(abc)\\k'1-0'", 3 => [:backref, :number_recursion_ref_sq, "\\k'1-0'", 5, 12]
52
-
53
- include_examples 'scan', '(abc)\k<+1-0>', 3 => [:backref, :name_recursion_ref_ab, '\k<+1-0>', 5, 13]
54
- include_examples 'scan', "(abc)\\k'+1-0'", 3 => [:backref, :name_recursion_ref_sq, "\\k'+1-0'", 5, 13]
55
- end
@@ -1,151 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe('Set scanning') do
4
- include_examples 'scan', /[a]/, 0 => [:set, :open, '[', 0, 1]
5
- include_examples 'scan', /[b]/, 2 => [:set, :close, ']', 2, 3]
6
- include_examples 'scan', /[^n]/, 1 => [:set, :negate, '^', 1, 2]
7
-
8
- include_examples 'scan', /[c]/, 1 => [:literal, :literal, 'c', 1, 2]
9
- include_examples 'scan', /[^d]/, 2 => [:literal, :literal, 'd', 2, 3]
10
-
11
- include_examples 'scan', /[\b]/, 1 => [:escape, :backspace, '\b', 1, 3]
12
- include_examples 'scan', /[A\bX]/, 2 => [:escape, :backspace, '\b', 2, 4]
13
-
14
- include_examples 'scan', /[\a]/, 1 => [:escape, :bell, '\a', 1, 3]
15
- include_examples 'scan', /[\e]/, 1 => [:escape, :escape, '\e', 1, 3]
16
- include_examples 'scan', /[\f]/, 1 => [:escape, :form_feed, '\f', 1, 3]
17
- include_examples 'scan', /[\n]/, 1 => [:escape, :newline, '\n', 1, 3]
18
- include_examples 'scan', /[\r]/, 1 => [:escape, :carriage, '\r', 1, 3]
19
- include_examples 'scan', /[\t]/, 1 => [:escape, :tab, '\t', 1, 3]
20
- include_examples 'scan', /[\v]/, 1 => [:escape, :vertical_tab, '\v', 1, 3]
21
-
22
- include_examples 'scan', /[.]/, 1 => [:literal, :literal, '.', 1, 2]
23
- include_examples 'scan', /[?]/, 1 => [:literal, :literal, '?', 1, 2]
24
- include_examples 'scan', /[*]/, 1 => [:literal, :literal, '*', 1, 2]
25
- include_examples 'scan', /[+]/, 1 => [:literal, :literal, '+', 1, 2]
26
- include_examples 'scan', /[{]/, 1 => [:literal, :literal, '{', 1, 2]
27
- include_examples 'scan', /[}]/, 1 => [:literal, :literal, '}', 1, 2]
28
- include_examples 'scan', /[<]/, 1 => [:literal, :literal, '<', 1, 2]
29
- include_examples 'scan', /[>]/, 1 => [:literal, :literal, '>', 1, 2]
30
-
31
- include_examples 'scan', '[\\\\]', 1 => [:escape, :backslash, '\\\\', 1, 3]
32
- include_examples 'scan', '[\u0040]', 1 => [:escape, :codepoint, '\u0040', 1, 7]
33
- include_examples 'scan', '[\u{40}]', 1 => [:escape, :codepoint_list, '\u{40}', 1, 7]
34
- include_examples 'scan', '[\c2]', 1 => [:escape, :control, '\c2', 1, 4]
35
- include_examples 'scan', '[\C-C]', 1 => [:escape, :control, '\C-C', 1, 5]
36
- include_examples 'scan', '[\x20]', 1 => [:escape, :hex, '\x20', 1, 5]
37
- include_examples 'scan', '[\M-Z]', 1 => [:escape, :meta_sequence, '\M-Z', 1, 5]
38
- include_examples 'scan', '[\M-\C-X]', 1 => [:escape, :meta_sequence, '\M-\C-X', 1, 8]
39
- include_examples 'scan', '[\\[]', 1 => [:escape, :set_open, '\[', 1, 3]
40
- include_examples 'scan', '[\\]]', 1 => [:escape, :set_close, '\]', 1, 3]
41
- include_examples 'scan', '[a\-]', 2 => [:escape, :literal, '\-', 2, 4]
42
- include_examples 'scan', '[\-c]', 1 => [:escape, :literal, '\-', 1, 3]
43
- include_examples 'scan', '[\.]', 1 => [:escape, :literal, '\.', 1, 3]
44
- include_examples 'scan', '[\?]', 1 => [:escape, :literal, '\?', 1, 3]
45
- include_examples 'scan', '[\*]', 1 => [:escape, :literal, '\*', 1, 3]
46
- include_examples 'scan', '[\+]', 1 => [:escape, :literal, '\+', 1, 3]
47
- include_examples 'scan', '[\|]', 1 => [:escape, :literal, '\|', 1, 3]
48
- include_examples 'scan', '[\{]', 1 => [:escape, :literal, '\{', 1, 3]
49
- include_examples 'scan', '[\}]', 1 => [:escape, :literal, '\}', 1, 3]
50
- include_examples 'scan', '[\(]', 1 => [:escape, :literal, '\(', 1, 3]
51
- include_examples 'scan', '[\)]', 1 => [:escape, :literal, '\)', 1, 3]
52
- include_examples 'scan', '[\!]', 1 => [:escape, :literal, '\!', 1, 3]
53
- include_examples 'scan', '[\#]', 1 => [:escape, :literal, '\#', 1, 3]
54
- include_examples 'scan', '[\A]', 1 => [:escape, :literal, '\A', 1, 3]
55
- include_examples 'scan', '[\z]', 1 => [:escape, :literal, '\z', 1, 3]
56
- include_examples 'scan', '[\g]', 1 => [:escape, :literal, '\g', 1, 3]
57
- include_examples 'scan', '[\K]', 1 => [:escape, :literal, '\K', 1, 3]
58
- include_examples 'scan', '[\R]', 1 => [:escape, :literal, '\R', 1, 3]
59
- include_examples 'scan', '[\X]', 1 => [:escape, :literal, '\X', 1, 3]
60
- include_examples 'scan', '[\B]', 1 => [:escape, :literal, '\B', 1, 3]
61
-
62
- include_examples 'scan', /[\d]/, 1 => [:type, :digit, '\d', 1, 3]
63
- include_examples 'scan', /[\da-z]/, 1 => [:type, :digit, '\d', 1, 3]
64
- include_examples 'scan', /[\D]/, 1 => [:type, :nondigit, '\D', 1, 3]
65
-
66
- include_examples 'scan', /[\h]/, 1 => [:type, :hex, '\h', 1, 3]
67
- include_examples 'scan', /[\H]/, 1 => [:type, :nonhex, '\H', 1, 3]
68
-
69
- include_examples 'scan', /[\s]/, 1 => [:type, :space, '\s', 1, 3]
70
- include_examples 'scan', /[\S]/, 1 => [:type, :nonspace, '\S', 1, 3]
71
-
72
- include_examples 'scan', /[\w]/, 1 => [:type, :word, '\w', 1, 3]
73
- include_examples 'scan', /[\W]/, 1 => [:type, :nonword, '\W', 1, 3]
74
-
75
- include_examples 'scan', /[a-b]/, 1 => [:literal, :literal, 'a', 1, 2]
76
- include_examples 'scan', /[a-c]/, 2 => [:set, :range, '-', 2, 3]
77
- include_examples 'scan', /[a-d]/, 3 => [:literal, :literal, 'd', 3, 4]
78
- include_examples 'scan', /[a-b-]/, 4 => [:literal, :literal, '-', 4, 5]
79
- include_examples 'scan', /[-a]/, 1 => [:literal, :literal, '-', 1, 2]
80
- include_examples 'scan', /[a-c^]/, 4 => [:literal, :literal, '^', 4, 5]
81
- include_examples 'scan', /[a-bd-f]/, 2 => [:set, :range, '-', 2, 3]
82
- include_examples 'scan', /[a-cd-f]/, 5 => [:set, :range, '-', 5, 6]
83
- # this is a buggy range, it matches only `c`, but not `a`, `b` or `-`
84
- include_examples 'scan', /[a-[c]]/, 2 => [:set, :range, '-', 2, 3]
85
- # these are not ranges, they match `a`, `c` and `-` (or non-`-` if negated)
86
- include_examples 'scan', /[[a]-[c]]/, 4 => [:literal, :literal, '-', 4, 5]
87
- include_examples 'scan', /[[a]-c]/, 4 => [:literal, :literal, '-', 4, 5]
88
- include_examples 'scan', /[^-c]/, 2 => [:literal, :literal, '-', 2, 3]
89
-
90
- include_examples 'scan', /[a[:digit:]c]/, 2 => [:posixclass, :digit, '[:digit:]', 2, 11]
91
- include_examples 'scan', /[[:digit:][:space:]]/, 2 => [:posixclass, :space, '[:space:]', 10, 19]
92
- include_examples 'scan', /[[:^digit:]]/, 1 => [:nonposixclass, :digit, '[:^digit:]', 1, 11]
93
-
94
- include_examples 'scan', /[a-d&&g-h]/, 4 => [:set, :intersection, '&&', 4, 6]
95
- include_examples 'scan', /[a&&]/, 2 => [:set, :intersection, '&&', 2, 4]
96
- include_examples 'scan', /[&&z]/, 1 => [:set, :intersection, '&&', 1, 3]
97
- include_examples 'scan', /[&&]/, 1 => [:set, :intersection, '&&', 1, 3]
98
-
99
- include_examples 'scan', /[a\p{digit}c]/, 2 => [:property, :digit, '\p{digit}', 2, 11]
100
- include_examples 'scan', /[a\P{digit}c]/, 2 => [:nonproperty, :digit, '\P{digit}', 2, 11]
101
- include_examples 'scan', /[a\p{^digit}c]/, 2 => [:nonproperty, :digit, '\p{^digit}', 2, 12]
102
- include_examples 'scan', /[a\P{^digit}c]/, 2 => [:property, :digit, '\P{^digit}', 2, 12]
103
-
104
- include_examples 'scan', /[a\p{ALPHA}c]/, 2 => [:property, :alpha, '\p{ALPHA}', 2, 11]
105
- include_examples 'scan', /[a\p{P}c]/, 2 => [:property, :punctuation,'\p{P}', 2, 7]
106
- include_examples 'scan', /[a\p{P}\P{P}c]/, 3 => [:nonproperty, :punctuation,'\P{P}', 7, 12]
107
-
108
- include_examples 'scan', /[\x20-\x27]/,
109
- 1 => [:escape, :hex, '\x20', 1, 5],
110
- 2 => [:set, :range, '-', 5, 6],
111
- 3 => [:escape, :hex, '\x27', 6, 10]
112
-
113
- include_examples 'scan', /[a-w&&[^c-g]z]/,
114
- 5 => [:set, :open, '[', 6, 7],
115
- 6 => [:set, :negate, '^', 7, 8],
116
- 8 => [:set, :range, '-', 9, 10],
117
- 10=> [:set, :close, ']', 11, 12]
118
-
119
- # Collations/collating sequences and character equivalents are not enabled
120
- # in Ruby at the moment. If they ever are, enable them in the scanner,
121
- # add them to a new syntax version, and handle them in the parser. Until then,
122
- # expect them to be scanned as regular subsets containing literals.
123
- # include_examples 'scan', /[a[.a-b.]c]/, 2 => [:set, :collation, '[.a-b.]', 2, 9]
124
- # include_examples 'scan', /[a[=e=]c]/, 2 => [:set, :equivalent, '[=e=]', 2, 7]
125
- include_examples 'scan', '[a[.a-b.]c]',
126
- 2 => [:set, :open, '[', 2, 3],
127
- 3 => [:literal, :literal, '.', 3, 4],
128
- 4 => [:literal, :literal, 'a', 4, 5]
129
- include_examples 'scan', '[a[=e=]c]',
130
- 2 => [:set, :open, '[', 2, 3],
131
- 3 => [:literal, :literal, '=', 3, 4],
132
- 4 => [:literal, :literal, 'e', 4, 5]
133
-
134
- # multi-byte characters should not affect indices
135
- include_examples 'scan', /[れます]/,
136
- 0 => [:set, :open, '[', 0, 1],
137
- 1 => [:literal, :literal, 'れ', 1, 2],
138
- 2 => [:literal, :literal, 'ま', 2, 3],
139
- 3 => [:literal, :literal, 'す', 3, 4],
140
- 4 => [:set, :close, ']', 4, 5]
141
-
142
- specify('set literal encoding') do
143
- text = RS.scan('[a]')[1][2].to_s
144
- expect(text).to eq 'a'
145
- expect(text.encoding.to_s).to eq 'UTF-8'
146
-
147
- text = RS.scan("[\u{1F632}]")[1][2].to_s
148
- expect(text).to eq "\u{1F632}"
149
- expect(text.encoding.to_s).to eq 'UTF-8'
150
- end
151
- end