regexp_parser 1.7.0 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +80 -1
  3. data/README.md +24 -12
  4. data/lib/regexp_parser/expression.rb +10 -19
  5. data/lib/regexp_parser/expression/classes/group.rb +17 -2
  6. data/lib/regexp_parser/expression/classes/root.rb +4 -16
  7. data/lib/regexp_parser/expression/quantifier.rb +9 -0
  8. data/lib/regexp_parser/expression/sequence.rb +0 -10
  9. data/lib/regexp_parser/lexer.rb +6 -6
  10. data/lib/regexp_parser/parser.rb +45 -12
  11. data/lib/regexp_parser/scanner.rb +1305 -1193
  12. data/lib/regexp_parser/scanner/char_type.rl +11 -11
  13. data/lib/regexp_parser/scanner/property.rl +2 -2
  14. data/lib/regexp_parser/scanner/scanner.rl +194 -171
  15. data/lib/regexp_parser/syntax/version_lookup.rb +2 -2
  16. data/lib/regexp_parser/version.rb +1 -1
  17. data/regexp_parser.gemspec +1 -1
  18. data/spec/expression/base_spec.rb +10 -0
  19. data/spec/expression/to_s_spec.rb +16 -0
  20. data/spec/lexer/delimiters_spec.rb +68 -0
  21. data/spec/lexer/literals_spec.rb +24 -49
  22. data/spec/parser/escapes_spec.rb +1 -1
  23. data/spec/parser/options_spec.rb +28 -0
  24. data/spec/parser/quantifiers_spec.rb +16 -0
  25. data/spec/parser/set/ranges_spec.rb +3 -3
  26. data/spec/scanner/delimiters_spec.rb +52 -0
  27. data/spec/scanner/errors_spec.rb +0 -1
  28. data/spec/scanner/escapes_spec.rb +10 -0
  29. data/spec/scanner/free_space_spec.rb +32 -0
  30. data/spec/scanner/literals_spec.rb +28 -38
  31. data/spec/scanner/options_spec.rb +36 -0
  32. data/spec/scanner/quantifiers_spec.rb +18 -13
  33. data/spec/scanner/sets_spec.rb +8 -2
  34. metadata +65 -61
  35. data/spec/expression/root_spec.rb +0 -9
  36. data/spec/expression/sequence_spec.rb +0 -9
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe('passing options to scan') do
4
+ def expect_type_tokens(tokens, type_tokens)
5
+ expect(tokens.map { |type, token, *| [type, token] }).to eq(type_tokens)
6
+ end
7
+
8
+ it 'raises if if scanning from a Regexp and options are passed' do
9
+ expect { RS.scan(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
10
+ ArgumentError,
11
+ 'options cannot be supplied unless scanning a String'
12
+ )
13
+ end
14
+
15
+ it 'sets free_spacing based on options if scanning from a String' do
16
+ expect_type_tokens(
17
+ RS.scan('a+#c', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED),
18
+ [
19
+ %i[literal literal],
20
+ %i[quantifier one_or_more],
21
+ %i[free_space comment]
22
+ ]
23
+ )
24
+ end
25
+
26
+ it 'does not set free_spacing if scanning from a String and passing no options' do
27
+ expect_type_tokens(
28
+ RS.scan('a+#c'),
29
+ [
30
+ %i[literal literal],
31
+ %i[quantifier one_or_more],
32
+ %i[literal literal]
33
+ ]
34
+ )
35
+ end
36
+ end
@@ -1,20 +1,25 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  RSpec.describe('Quantifier scanning') do
4
- include_examples 'scan', 'a?', 1 => [:quantifier, :zero_or_one, '?', 1, 2]
5
- include_examples 'scan', 'a??', 1 => [:quantifier, :zero_or_one_reluctant, '??', 1, 3]
6
- include_examples 'scan', 'a?+', 1 => [:quantifier, :zero_or_one_possessive, '?+', 1, 3]
4
+ include_examples 'scan', 'a?', 1 => [:quantifier, :zero_or_one, '?', 1, 2]
5
+ include_examples 'scan', 'a??', 1 => [:quantifier, :zero_or_one_reluctant, '??', 1, 3]
6
+ include_examples 'scan', 'a?+', 1 => [:quantifier, :zero_or_one_possessive, '?+', 1, 3]
7
7
 
8
- include_examples 'scan', 'a*', 1 => [:quantifier, :zero_or_more, '*', 1, 2]
9
- include_examples 'scan', 'a*?', 1 => [:quantifier, :zero_or_more_reluctant, '*?', 1, 3]
10
- include_examples 'scan', 'a*+', 1 => [:quantifier, :zero_or_more_possessive, '*+', 1, 3]
8
+ include_examples 'scan', 'a*', 1 => [:quantifier, :zero_or_more, '*', 1, 2]
9
+ include_examples 'scan', 'a*?', 1 => [:quantifier, :zero_or_more_reluctant, '*?', 1, 3]
10
+ include_examples 'scan', 'a*+', 1 => [:quantifier, :zero_or_more_possessive, '*+', 1, 3]
11
11
 
12
- include_examples 'scan', 'a+', 1 => [:quantifier, :one_or_more, '+', 1, 2]
13
- include_examples 'scan', 'a+?', 1 => [:quantifier, :one_or_more_reluctant, '+?', 1, 3]
14
- include_examples 'scan', 'a++', 1 => [:quantifier, :one_or_more_possessive, '++', 1, 3]
12
+ include_examples 'scan', 'a+', 1 => [:quantifier, :one_or_more, '+', 1, 2]
13
+ include_examples 'scan', 'a+?', 1 => [:quantifier, :one_or_more_reluctant, '+?', 1, 3]
14
+ include_examples 'scan', 'a++', 1 => [:quantifier, :one_or_more_possessive, '++', 1, 3]
15
15
 
16
- include_examples 'scan', 'a{2}', 1 => [:quantifier, :interval, '{2}', 1, 4]
17
- include_examples 'scan', 'a{2,}', 1 => [:quantifier, :interval, '{2,}', 1, 5]
18
- include_examples 'scan', 'a{,2}', 1 => [:quantifier, :interval, '{,2}', 1, 5]
19
- include_examples 'scan', 'a{2,4}', 1 => [:quantifier, :interval, '{2,4}', 1, 6]
16
+ include_examples 'scan', 'a{2}', 1 => [:quantifier, :interval, '{2}', 1, 4]
17
+ include_examples 'scan', 'a{2,}', 1 => [:quantifier, :interval, '{2,}', 1, 5]
18
+ include_examples 'scan', 'a{,2}', 1 => [:quantifier, :interval, '{,2}', 1, 5]
19
+ include_examples 'scan', 'a{2,4}', 1 => [:quantifier, :interval, '{2,4}', 1, 6]
20
+
21
+ # special case: chained quantifiers
22
+ include_examples 'scan', 'a+{2}{3}', 1 => [:quantifier, :one_or_more, '+', 1, 2]
23
+ include_examples 'scan', 'a+{2}{3}', 2 => [:quantifier, :interval, '{2}', 2, 5]
24
+ include_examples 'scan', 'a+{2}{3}', 3 => [:quantifier, :interval, '{3}', 5, 8]
20
25
  end
@@ -18,8 +18,6 @@ RSpec.describe('Set scanning') do
18
18
  include_examples 'scan', /[<]/, 1 => [:literal, :literal, '<', 1, 2]
19
19
  include_examples 'scan', /[>]/, 1 => [:literal, :literal, '>', 1, 2]
20
20
 
21
- include_examples 'scan', /[äöü]/, 2 => [:literal, :literal, 'ö', 3, 5]
22
-
23
21
  include_examples 'scan', /[\x20]/, 1 => [:escape, :hex, '\x20', 1, 5]
24
22
 
25
23
  include_examples 'scan', '[\.]', 1 => [:escape, :dot, '\.', 1, 3]
@@ -90,6 +88,14 @@ RSpec.describe('Set scanning') do
90
88
  8 => [:set, :range, '-', 9, 10],
91
89
  10=> [:set, :close, ']', 11, 12]
92
90
 
91
+ # multi-byte characters should not affect indices
92
+ include_examples 'scan', /[れます]/,
93
+ 0 => [:set, :open, '[', 0, 1],
94
+ 1 => [:literal, :literal, 'れ', 1, 2],
95
+ 2 => [:literal, :literal, 'ま', 2, 3],
96
+ 3 => [:literal, :literal, 'す', 3, 4],
97
+ 4 => [:set, :close, ']', 4, 5]
98
+
93
99
  specify('set literal encoding') do
94
100
  text = RS.scan('[a]')[1][2].to_s
95
101
  expect(text).to eq 'a'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: regexp_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ammar Ali
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-23 00:00:00.000000000 Z
11
+ date: 2020-11-25 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: A library for tokenizing, lexing, and parsing Ruby regular expressions.
14
14
  email:
@@ -102,13 +102,12 @@ files:
102
102
  - spec/expression/methods/tests_spec.rb
103
103
  - spec/expression/methods/traverse_spec.rb
104
104
  - spec/expression/options_spec.rb
105
- - spec/expression/root_spec.rb
106
- - spec/expression/sequence_spec.rb
107
105
  - spec/expression/subexpression_spec.rb
108
106
  - spec/expression/to_h_spec.rb
109
107
  - spec/expression/to_s_spec.rb
110
108
  - spec/lexer/all_spec.rb
111
109
  - spec/lexer/conditionals_spec.rb
110
+ - spec/lexer/delimiters_spec.rb
112
111
  - spec/lexer/escapes_spec.rb
113
112
  - spec/lexer/keep_spec.rb
114
113
  - spec/lexer/literals_spec.rb
@@ -123,6 +122,7 @@ files:
123
122
  - spec/parser/free_space_spec.rb
124
123
  - spec/parser/groups_spec.rb
125
124
  - spec/parser/keep_spec.rb
125
+ - spec/parser/options_spec.rb
126
126
  - spec/parser/posix_classes_spec.rb
127
127
  - spec/parser/properties_spec.rb
128
128
  - spec/parser/quantifiers_spec.rb
@@ -134,6 +134,7 @@ files:
134
134
  - spec/scanner/all_spec.rb
135
135
  - spec/scanner/anchors_spec.rb
136
136
  - spec/scanner/conditionals_spec.rb
137
+ - spec/scanner/delimiters_spec.rb
137
138
  - spec/scanner/errors_spec.rb
138
139
  - spec/scanner/escapes_spec.rb
139
140
  - spec/scanner/free_space_spec.rb
@@ -141,6 +142,7 @@ files:
141
142
  - spec/scanner/keep_spec.rb
142
143
  - spec/scanner/literals_spec.rb
143
144
  - spec/scanner/meta_spec.rb
145
+ - spec/scanner/options_spec.rb
144
146
  - spec/scanner/properties_spec.rb
145
147
  - spec/scanner/quantifiers_spec.rb
146
148
  - spec/scanner/refcalls_spec.rb
@@ -164,7 +166,7 @@ licenses:
164
166
  - MIT
165
167
  metadata:
166
168
  issue_tracker: https://github.com/ammar/regexp_parser/issues
167
- post_install_message:
169
+ post_install_message:
168
170
  rdoc_options:
169
171
  - "--inline-source"
170
172
  - "--charset=UTF-8"
@@ -174,82 +176,84 @@ required_ruby_version: !ruby/object:Gem::Requirement
174
176
  requirements:
175
177
  - - ">="
176
178
  - !ruby/object:Gem::Version
177
- version: 1.9.1
179
+ version: 2.0.0
178
180
  required_rubygems_version: !ruby/object:Gem::Requirement
179
181
  requirements:
180
182
  - - ">="
181
183
  - !ruby/object:Gem::Version
182
184
  version: '0'
183
185
  requirements: []
184
- rubygems_version: 3.1.2
185
- signing_key:
186
+ rubygems_version: 3.2.0.rc.1
187
+ signing_key:
186
188
  specification_version: 4
187
189
  summary: Scanner, lexer, parser for ruby's regular expressions
188
190
  test_files:
189
- - spec/token/token_spec.rb
190
- - spec/spec_helper.rb
191
- - spec/lexer/escapes_spec.rb
192
- - spec/lexer/keep_spec.rb
191
+ - spec/expression/base_spec.rb
192
+ - spec/expression/clone_spec.rb
193
+ - spec/expression/conditional_spec.rb
194
+ - spec/expression/free_space_spec.rb
195
+ - spec/expression/methods/match_length_spec.rb
196
+ - spec/expression/methods/match_spec.rb
197
+ - spec/expression/methods/strfregexp_spec.rb
198
+ - spec/expression/methods/tests_spec.rb
199
+ - spec/expression/methods/traverse_spec.rb
200
+ - spec/expression/options_spec.rb
201
+ - spec/expression/subexpression_spec.rb
202
+ - spec/expression/to_h_spec.rb
203
+ - spec/expression/to_s_spec.rb
193
204
  - spec/lexer/all_spec.rb
194
205
  - spec/lexer/conditionals_spec.rb
206
+ - spec/lexer/delimiters_spec.rb
207
+ - spec/lexer/escapes_spec.rb
208
+ - spec/lexer/keep_spec.rb
209
+ - spec/lexer/literals_spec.rb
195
210
  - spec/lexer/nesting_spec.rb
196
211
  - spec/lexer/refcalls_spec.rb
197
- - spec/lexer/literals_spec.rb
198
- - spec/parser/escapes_spec.rb
199
- - spec/parser/properties_spec.rb
200
- - spec/parser/sets_spec.rb
201
- - spec/parser/free_space_spec.rb
202
- - spec/parser/keep_spec.rb
203
212
  - spec/parser/all_spec.rb
204
- - spec/parser/conditionals_spec.rb
205
- - spec/parser/types_spec.rb
206
- - spec/parser/anchors_spec.rb
207
213
  - spec/parser/alternation_spec.rb
208
- - spec/parser/posix_classes_spec.rb
209
- - spec/parser/set/ranges_spec.rb
210
- - spec/parser/set/intersections_spec.rb
214
+ - spec/parser/anchors_spec.rb
215
+ - spec/parser/conditionals_spec.rb
211
216
  - spec/parser/errors_spec.rb
212
- - spec/parser/refcalls_spec.rb
217
+ - spec/parser/escapes_spec.rb
218
+ - spec/parser/free_space_spec.rb
213
219
  - spec/parser/groups_spec.rb
220
+ - spec/parser/keep_spec.rb
221
+ - spec/parser/options_spec.rb
222
+ - spec/parser/posix_classes_spec.rb
223
+ - spec/parser/properties_spec.rb
214
224
  - spec/parser/quantifiers_spec.rb
215
- - spec/support/warning_extractor.rb
216
- - spec/support/shared_examples.rb
217
- - spec/support/runner.rb
218
- - spec/expression/subexpression_spec.rb
219
- - spec/expression/methods/match_spec.rb
220
- - spec/expression/methods/match_length_spec.rb
221
- - spec/expression/methods/traverse_spec.rb
222
- - spec/expression/methods/strfregexp_spec.rb
223
- - spec/expression/methods/tests_spec.rb
224
- - spec/expression/free_space_spec.rb
225
- - spec/expression/options_spec.rb
226
- - spec/expression/to_s_spec.rb
227
- - spec/expression/root_spec.rb
228
- - spec/expression/sequence_spec.rb
229
- - spec/expression/clone_spec.rb
230
- - spec/expression/to_h_spec.rb
231
- - spec/expression/conditional_spec.rb
232
- - spec/expression/base_spec.rb
233
- - spec/syntax/syntax_spec.rb
234
- - spec/syntax/syntax_token_map_spec.rb
235
- - spec/syntax/versions/1.9.3_spec.rb
236
- - spec/syntax/versions/2.2.0_spec.rb
237
- - spec/syntax/versions/1.9.1_spec.rb
238
- - spec/syntax/versions/2.0.0_spec.rb
239
- - spec/syntax/versions/1.8.6_spec.rb
240
- - spec/syntax/versions/aliases_spec.rb
241
- - spec/scanner/escapes_spec.rb
242
- - spec/scanner/properties_spec.rb
243
- - spec/scanner/sets_spec.rb
244
- - spec/scanner/free_space_spec.rb
245
- - spec/scanner/keep_spec.rb
225
+ - spec/parser/refcalls_spec.rb
226
+ - spec/parser/set/intersections_spec.rb
227
+ - spec/parser/set/ranges_spec.rb
228
+ - spec/parser/sets_spec.rb
229
+ - spec/parser/types_spec.rb
246
230
  - spec/scanner/all_spec.rb
247
- - spec/scanner/conditionals_spec.rb
248
- - spec/scanner/types_spec.rb
249
231
  - spec/scanner/anchors_spec.rb
250
- - spec/scanner/meta_spec.rb
232
+ - spec/scanner/conditionals_spec.rb
233
+ - spec/scanner/delimiters_spec.rb
251
234
  - spec/scanner/errors_spec.rb
252
- - spec/scanner/refcalls_spec.rb
235
+ - spec/scanner/escapes_spec.rb
236
+ - spec/scanner/free_space_spec.rb
253
237
  - spec/scanner/groups_spec.rb
238
+ - spec/scanner/keep_spec.rb
254
239
  - spec/scanner/literals_spec.rb
240
+ - spec/scanner/meta_spec.rb
241
+ - spec/scanner/options_spec.rb
242
+ - spec/scanner/properties_spec.rb
255
243
  - spec/scanner/quantifiers_spec.rb
244
+ - spec/scanner/refcalls_spec.rb
245
+ - spec/scanner/sets_spec.rb
246
+ - spec/scanner/types_spec.rb
247
+ - spec/spec_helper.rb
248
+ - spec/support/runner.rb
249
+ - spec/support/shared_examples.rb
250
+ - spec/support/warning_extractor.rb
251
+ - spec/syntax/syntax_spec.rb
252
+ - spec/syntax/syntax_token_map_spec.rb
253
+ - spec/syntax/versions/1.8.6_spec.rb
254
+ - spec/syntax/versions/1.9.1_spec.rb
255
+ - spec/syntax/versions/1.9.3_spec.rb
256
+ - spec/syntax/versions/2.0.0_spec.rb
257
+ - spec/syntax/versions/2.2.0_spec.rb
258
+ - spec/syntax/versions/aliases_spec.rb
259
+ - spec/token/token_spec.rb
@@ -1,9 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe(Regexp::Expression::Root) do
4
- describe('#initialize') do
5
- it 'supports the old, nonstandard arity for backwards compatibility' do
6
- expect { Root.new }.to output.to_stderr
7
- end
8
- end
9
- end
@@ -1,9 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe(Regexp::Expression::Sequence) do
4
- describe('#initialize') do
5
- it 'supports the old, nonstandard arity for backwards compatibility' do
6
- expect { Sequence.new(0, 0, 0) }.to output.to_stderr
7
- end
8
- end
9
- end