regexp_parser 1.7.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +80 -1
  3. data/README.md +24 -12
  4. data/lib/regexp_parser/expression.rb +10 -19
  5. data/lib/regexp_parser/expression/classes/group.rb +17 -2
  6. data/lib/regexp_parser/expression/classes/root.rb +4 -16
  7. data/lib/regexp_parser/expression/quantifier.rb +9 -0
  8. data/lib/regexp_parser/expression/sequence.rb +0 -10
  9. data/lib/regexp_parser/lexer.rb +6 -6
  10. data/lib/regexp_parser/parser.rb +45 -12
  11. data/lib/regexp_parser/scanner.rb +1305 -1193
  12. data/lib/regexp_parser/scanner/char_type.rl +11 -11
  13. data/lib/regexp_parser/scanner/property.rl +2 -2
  14. data/lib/regexp_parser/scanner/scanner.rl +194 -171
  15. data/lib/regexp_parser/syntax/version_lookup.rb +2 -2
  16. data/lib/regexp_parser/version.rb +1 -1
  17. data/regexp_parser.gemspec +1 -1
  18. data/spec/expression/base_spec.rb +10 -0
  19. data/spec/expression/to_s_spec.rb +16 -0
  20. data/spec/lexer/delimiters_spec.rb +68 -0
  21. data/spec/lexer/literals_spec.rb +24 -49
  22. data/spec/parser/escapes_spec.rb +1 -1
  23. data/spec/parser/options_spec.rb +28 -0
  24. data/spec/parser/quantifiers_spec.rb +16 -0
  25. data/spec/parser/set/ranges_spec.rb +3 -3
  26. data/spec/scanner/delimiters_spec.rb +52 -0
  27. data/spec/scanner/errors_spec.rb +0 -1
  28. data/spec/scanner/escapes_spec.rb +10 -0
  29. data/spec/scanner/free_space_spec.rb +32 -0
  30. data/spec/scanner/literals_spec.rb +28 -38
  31. data/spec/scanner/options_spec.rb +36 -0
  32. data/spec/scanner/quantifiers_spec.rb +18 -13
  33. data/spec/scanner/sets_spec.rb +8 -2
  34. metadata +65 -61
  35. data/spec/expression/root_spec.rb +0 -9
  36. data/spec/expression/sequence_spec.rb +0 -9
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe('passing options to scan') do
4
+ def expect_type_tokens(tokens, type_tokens)
5
+ expect(tokens.map { |type, token, *| [type, token] }).to eq(type_tokens)
6
+ end
7
+
8
+ it 'raises if if scanning from a Regexp and options are passed' do
9
+ expect { RS.scan(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
10
+ ArgumentError,
11
+ 'options cannot be supplied unless scanning a String'
12
+ )
13
+ end
14
+
15
+ it 'sets free_spacing based on options if scanning from a String' do
16
+ expect_type_tokens(
17
+ RS.scan('a+#c', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED),
18
+ [
19
+ %i[literal literal],
20
+ %i[quantifier one_or_more],
21
+ %i[free_space comment]
22
+ ]
23
+ )
24
+ end
25
+
26
+ it 'does not set free_spacing if scanning from a String and passing no options' do
27
+ expect_type_tokens(
28
+ RS.scan('a+#c'),
29
+ [
30
+ %i[literal literal],
31
+ %i[quantifier one_or_more],
32
+ %i[literal literal]
33
+ ]
34
+ )
35
+ end
36
+ end
@@ -1,20 +1,25 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  RSpec.describe('Quantifier scanning') do
4
- include_examples 'scan', 'a?', 1 => [:quantifier, :zero_or_one, '?', 1, 2]
5
- include_examples 'scan', 'a??', 1 => [:quantifier, :zero_or_one_reluctant, '??', 1, 3]
6
- include_examples 'scan', 'a?+', 1 => [:quantifier, :zero_or_one_possessive, '?+', 1, 3]
4
+ include_examples 'scan', 'a?', 1 => [:quantifier, :zero_or_one, '?', 1, 2]
5
+ include_examples 'scan', 'a??', 1 => [:quantifier, :zero_or_one_reluctant, '??', 1, 3]
6
+ include_examples 'scan', 'a?+', 1 => [:quantifier, :zero_or_one_possessive, '?+', 1, 3]
7
7
 
8
- include_examples 'scan', 'a*', 1 => [:quantifier, :zero_or_more, '*', 1, 2]
9
- include_examples 'scan', 'a*?', 1 => [:quantifier, :zero_or_more_reluctant, '*?', 1, 3]
10
- include_examples 'scan', 'a*+', 1 => [:quantifier, :zero_or_more_possessive, '*+', 1, 3]
8
+ include_examples 'scan', 'a*', 1 => [:quantifier, :zero_or_more, '*', 1, 2]
9
+ include_examples 'scan', 'a*?', 1 => [:quantifier, :zero_or_more_reluctant, '*?', 1, 3]
10
+ include_examples 'scan', 'a*+', 1 => [:quantifier, :zero_or_more_possessive, '*+', 1, 3]
11
11
 
12
- include_examples 'scan', 'a+', 1 => [:quantifier, :one_or_more, '+', 1, 2]
13
- include_examples 'scan', 'a+?', 1 => [:quantifier, :one_or_more_reluctant, '+?', 1, 3]
14
- include_examples 'scan', 'a++', 1 => [:quantifier, :one_or_more_possessive, '++', 1, 3]
12
+ include_examples 'scan', 'a+', 1 => [:quantifier, :one_or_more, '+', 1, 2]
13
+ include_examples 'scan', 'a+?', 1 => [:quantifier, :one_or_more_reluctant, '+?', 1, 3]
14
+ include_examples 'scan', 'a++', 1 => [:quantifier, :one_or_more_possessive, '++', 1, 3]
15
15
 
16
- include_examples 'scan', 'a{2}', 1 => [:quantifier, :interval, '{2}', 1, 4]
17
- include_examples 'scan', 'a{2,}', 1 => [:quantifier, :interval, '{2,}', 1, 5]
18
- include_examples 'scan', 'a{,2}', 1 => [:quantifier, :interval, '{,2}', 1, 5]
19
- include_examples 'scan', 'a{2,4}', 1 => [:quantifier, :interval, '{2,4}', 1, 6]
16
+ include_examples 'scan', 'a{2}', 1 => [:quantifier, :interval, '{2}', 1, 4]
17
+ include_examples 'scan', 'a{2,}', 1 => [:quantifier, :interval, '{2,}', 1, 5]
18
+ include_examples 'scan', 'a{,2}', 1 => [:quantifier, :interval, '{,2}', 1, 5]
19
+ include_examples 'scan', 'a{2,4}', 1 => [:quantifier, :interval, '{2,4}', 1, 6]
20
+
21
+ # special case: chained quantifiers
22
+ include_examples 'scan', 'a+{2}{3}', 1 => [:quantifier, :one_or_more, '+', 1, 2]
23
+ include_examples 'scan', 'a+{2}{3}', 2 => [:quantifier, :interval, '{2}', 2, 5]
24
+ include_examples 'scan', 'a+{2}{3}', 3 => [:quantifier, :interval, '{3}', 5, 8]
20
25
  end
@@ -18,8 +18,6 @@ RSpec.describe('Set scanning') do
18
18
  include_examples 'scan', /[<]/, 1 => [:literal, :literal, '<', 1, 2]
19
19
  include_examples 'scan', /[>]/, 1 => [:literal, :literal, '>', 1, 2]
20
20
 
21
- include_examples 'scan', /[äöü]/, 2 => [:literal, :literal, 'ö', 3, 5]
22
-
23
21
  include_examples 'scan', /[\x20]/, 1 => [:escape, :hex, '\x20', 1, 5]
24
22
 
25
23
  include_examples 'scan', '[\.]', 1 => [:escape, :dot, '\.', 1, 3]
@@ -90,6 +88,14 @@ RSpec.describe('Set scanning') do
90
88
  8 => [:set, :range, '-', 9, 10],
91
89
  10=> [:set, :close, ']', 11, 12]
92
90
 
91
+ # multi-byte characters should not affect indices
92
+ include_examples 'scan', /[れます]/,
93
+ 0 => [:set, :open, '[', 0, 1],
94
+ 1 => [:literal, :literal, 'れ', 1, 2],
95
+ 2 => [:literal, :literal, 'ま', 2, 3],
96
+ 3 => [:literal, :literal, 'す', 3, 4],
97
+ 4 => [:set, :close, ']', 4, 5]
98
+
93
99
  specify('set literal encoding') do
94
100
  text = RS.scan('[a]')[1][2].to_s
95
101
  expect(text).to eq 'a'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: regexp_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ammar Ali
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-23 00:00:00.000000000 Z
11
+ date: 2020-11-25 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: A library for tokenizing, lexing, and parsing Ruby regular expressions.
14
14
  email:
@@ -102,13 +102,12 @@ files:
102
102
  - spec/expression/methods/tests_spec.rb
103
103
  - spec/expression/methods/traverse_spec.rb
104
104
  - spec/expression/options_spec.rb
105
- - spec/expression/root_spec.rb
106
- - spec/expression/sequence_spec.rb
107
105
  - spec/expression/subexpression_spec.rb
108
106
  - spec/expression/to_h_spec.rb
109
107
  - spec/expression/to_s_spec.rb
110
108
  - spec/lexer/all_spec.rb
111
109
  - spec/lexer/conditionals_spec.rb
110
+ - spec/lexer/delimiters_spec.rb
112
111
  - spec/lexer/escapes_spec.rb
113
112
  - spec/lexer/keep_spec.rb
114
113
  - spec/lexer/literals_spec.rb
@@ -123,6 +122,7 @@ files:
123
122
  - spec/parser/free_space_spec.rb
124
123
  - spec/parser/groups_spec.rb
125
124
  - spec/parser/keep_spec.rb
125
+ - spec/parser/options_spec.rb
126
126
  - spec/parser/posix_classes_spec.rb
127
127
  - spec/parser/properties_spec.rb
128
128
  - spec/parser/quantifiers_spec.rb
@@ -134,6 +134,7 @@ files:
134
134
  - spec/scanner/all_spec.rb
135
135
  - spec/scanner/anchors_spec.rb
136
136
  - spec/scanner/conditionals_spec.rb
137
+ - spec/scanner/delimiters_spec.rb
137
138
  - spec/scanner/errors_spec.rb
138
139
  - spec/scanner/escapes_spec.rb
139
140
  - spec/scanner/free_space_spec.rb
@@ -141,6 +142,7 @@ files:
141
142
  - spec/scanner/keep_spec.rb
142
143
  - spec/scanner/literals_spec.rb
143
144
  - spec/scanner/meta_spec.rb
145
+ - spec/scanner/options_spec.rb
144
146
  - spec/scanner/properties_spec.rb
145
147
  - spec/scanner/quantifiers_spec.rb
146
148
  - spec/scanner/refcalls_spec.rb
@@ -164,7 +166,7 @@ licenses:
164
166
  - MIT
165
167
  metadata:
166
168
  issue_tracker: https://github.com/ammar/regexp_parser/issues
167
- post_install_message:
169
+ post_install_message:
168
170
  rdoc_options:
169
171
  - "--inline-source"
170
172
  - "--charset=UTF-8"
@@ -174,82 +176,84 @@ required_ruby_version: !ruby/object:Gem::Requirement
174
176
  requirements:
175
177
  - - ">="
176
178
  - !ruby/object:Gem::Version
177
- version: 1.9.1
179
+ version: 2.0.0
178
180
  required_rubygems_version: !ruby/object:Gem::Requirement
179
181
  requirements:
180
182
  - - ">="
181
183
  - !ruby/object:Gem::Version
182
184
  version: '0'
183
185
  requirements: []
184
- rubygems_version: 3.1.2
185
- signing_key:
186
+ rubygems_version: 3.2.0.rc.1
187
+ signing_key:
186
188
  specification_version: 4
187
189
  summary: Scanner, lexer, parser for ruby's regular expressions
188
190
  test_files:
189
- - spec/token/token_spec.rb
190
- - spec/spec_helper.rb
191
- - spec/lexer/escapes_spec.rb
192
- - spec/lexer/keep_spec.rb
191
+ - spec/expression/base_spec.rb
192
+ - spec/expression/clone_spec.rb
193
+ - spec/expression/conditional_spec.rb
194
+ - spec/expression/free_space_spec.rb
195
+ - spec/expression/methods/match_length_spec.rb
196
+ - spec/expression/methods/match_spec.rb
197
+ - spec/expression/methods/strfregexp_spec.rb
198
+ - spec/expression/methods/tests_spec.rb
199
+ - spec/expression/methods/traverse_spec.rb
200
+ - spec/expression/options_spec.rb
201
+ - spec/expression/subexpression_spec.rb
202
+ - spec/expression/to_h_spec.rb
203
+ - spec/expression/to_s_spec.rb
193
204
  - spec/lexer/all_spec.rb
194
205
  - spec/lexer/conditionals_spec.rb
206
+ - spec/lexer/delimiters_spec.rb
207
+ - spec/lexer/escapes_spec.rb
208
+ - spec/lexer/keep_spec.rb
209
+ - spec/lexer/literals_spec.rb
195
210
  - spec/lexer/nesting_spec.rb
196
211
  - spec/lexer/refcalls_spec.rb
197
- - spec/lexer/literals_spec.rb
198
- - spec/parser/escapes_spec.rb
199
- - spec/parser/properties_spec.rb
200
- - spec/parser/sets_spec.rb
201
- - spec/parser/free_space_spec.rb
202
- - spec/parser/keep_spec.rb
203
212
  - spec/parser/all_spec.rb
204
- - spec/parser/conditionals_spec.rb
205
- - spec/parser/types_spec.rb
206
- - spec/parser/anchors_spec.rb
207
213
  - spec/parser/alternation_spec.rb
208
- - spec/parser/posix_classes_spec.rb
209
- - spec/parser/set/ranges_spec.rb
210
- - spec/parser/set/intersections_spec.rb
214
+ - spec/parser/anchors_spec.rb
215
+ - spec/parser/conditionals_spec.rb
211
216
  - spec/parser/errors_spec.rb
212
- - spec/parser/refcalls_spec.rb
217
+ - spec/parser/escapes_spec.rb
218
+ - spec/parser/free_space_spec.rb
213
219
  - spec/parser/groups_spec.rb
220
+ - spec/parser/keep_spec.rb
221
+ - spec/parser/options_spec.rb
222
+ - spec/parser/posix_classes_spec.rb
223
+ - spec/parser/properties_spec.rb
214
224
  - spec/parser/quantifiers_spec.rb
215
- - spec/support/warning_extractor.rb
216
- - spec/support/shared_examples.rb
217
- - spec/support/runner.rb
218
- - spec/expression/subexpression_spec.rb
219
- - spec/expression/methods/match_spec.rb
220
- - spec/expression/methods/match_length_spec.rb
221
- - spec/expression/methods/traverse_spec.rb
222
- - spec/expression/methods/strfregexp_spec.rb
223
- - spec/expression/methods/tests_spec.rb
224
- - spec/expression/free_space_spec.rb
225
- - spec/expression/options_spec.rb
226
- - spec/expression/to_s_spec.rb
227
- - spec/expression/root_spec.rb
228
- - spec/expression/sequence_spec.rb
229
- - spec/expression/clone_spec.rb
230
- - spec/expression/to_h_spec.rb
231
- - spec/expression/conditional_spec.rb
232
- - spec/expression/base_spec.rb
233
- - spec/syntax/syntax_spec.rb
234
- - spec/syntax/syntax_token_map_spec.rb
235
- - spec/syntax/versions/1.9.3_spec.rb
236
- - spec/syntax/versions/2.2.0_spec.rb
237
- - spec/syntax/versions/1.9.1_spec.rb
238
- - spec/syntax/versions/2.0.0_spec.rb
239
- - spec/syntax/versions/1.8.6_spec.rb
240
- - spec/syntax/versions/aliases_spec.rb
241
- - spec/scanner/escapes_spec.rb
242
- - spec/scanner/properties_spec.rb
243
- - spec/scanner/sets_spec.rb
244
- - spec/scanner/free_space_spec.rb
245
- - spec/scanner/keep_spec.rb
225
+ - spec/parser/refcalls_spec.rb
226
+ - spec/parser/set/intersections_spec.rb
227
+ - spec/parser/set/ranges_spec.rb
228
+ - spec/parser/sets_spec.rb
229
+ - spec/parser/types_spec.rb
246
230
  - spec/scanner/all_spec.rb
247
- - spec/scanner/conditionals_spec.rb
248
- - spec/scanner/types_spec.rb
249
231
  - spec/scanner/anchors_spec.rb
250
- - spec/scanner/meta_spec.rb
232
+ - spec/scanner/conditionals_spec.rb
233
+ - spec/scanner/delimiters_spec.rb
251
234
  - spec/scanner/errors_spec.rb
252
- - spec/scanner/refcalls_spec.rb
235
+ - spec/scanner/escapes_spec.rb
236
+ - spec/scanner/free_space_spec.rb
253
237
  - spec/scanner/groups_spec.rb
238
+ - spec/scanner/keep_spec.rb
254
239
  - spec/scanner/literals_spec.rb
240
+ - spec/scanner/meta_spec.rb
241
+ - spec/scanner/options_spec.rb
242
+ - spec/scanner/properties_spec.rb
255
243
  - spec/scanner/quantifiers_spec.rb
244
+ - spec/scanner/refcalls_spec.rb
245
+ - spec/scanner/sets_spec.rb
246
+ - spec/scanner/types_spec.rb
247
+ - spec/spec_helper.rb
248
+ - spec/support/runner.rb
249
+ - spec/support/shared_examples.rb
250
+ - spec/support/warning_extractor.rb
251
+ - spec/syntax/syntax_spec.rb
252
+ - spec/syntax/syntax_token_map_spec.rb
253
+ - spec/syntax/versions/1.8.6_spec.rb
254
+ - spec/syntax/versions/1.9.1_spec.rb
255
+ - spec/syntax/versions/1.9.3_spec.rb
256
+ - spec/syntax/versions/2.0.0_spec.rb
257
+ - spec/syntax/versions/2.2.0_spec.rb
258
+ - spec/syntax/versions/aliases_spec.rb
259
+ - spec/token/token_spec.rb
@@ -1,9 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe(Regexp::Expression::Root) do
4
- describe('#initialize') do
5
- it 'supports the old, nonstandard arity for backwards compatibility' do
6
- expect { Root.new }.to output.to_stderr
7
- end
8
- end
9
- end
@@ -1,9 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe(Regexp::Expression::Sequence) do
4
- describe('#initialize') do
5
- it 'supports the old, nonstandard arity for backwards compatibility' do
6
- expect { Sequence.new(0, 0, 0) }.to output.to_stderr
7
- end
8
- end
9
- end