regexp_parser 2.7.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/LICENSE +1 -1
- data/Rakefile +5 -3
- data/lib/regexp_parser/error.rb +2 -0
- data/lib/regexp_parser/expression/base.rb +2 -7
- data/lib/regexp_parser/expression/classes/alternation.rb +3 -1
- data/lib/regexp_parser/expression/classes/anchor.rb +2 -0
- data/lib/regexp_parser/expression/classes/backreference.rb +7 -26
- data/lib/regexp_parser/expression/classes/character_set/intersection.rb +2 -0
- data/lib/regexp_parser/expression/classes/character_set/range.rb +4 -7
- data/lib/regexp_parser/expression/classes/character_set.rb +6 -8
- data/lib/regexp_parser/expression/classes/character_type.rb +2 -0
- data/lib/regexp_parser/expression/classes/conditional.rb +4 -28
- data/lib/regexp_parser/expression/classes/escape_sequence.rb +29 -96
- data/lib/regexp_parser/expression/classes/free_space.rb +5 -1
- data/lib/regexp_parser/expression/classes/group.rb +2 -22
- data/lib/regexp_parser/expression/classes/keep.rb +3 -1
- data/lib/regexp_parser/expression/classes/literal.rb +2 -0
- data/lib/regexp_parser/expression/classes/posix_class.rb +7 -5
- data/lib/regexp_parser/expression/classes/root.rb +2 -0
- data/lib/regexp_parser/expression/classes/unicode_property.rb +13 -11
- data/lib/regexp_parser/expression/methods/construct.rb +4 -4
- data/lib/regexp_parser/expression/methods/escape_sequence_char.rb +7 -0
- data/lib/regexp_parser/expression/methods/escape_sequence_codepoint.rb +76 -0
- data/lib/regexp_parser/expression/methods/human_name.rb +2 -0
- data/lib/regexp_parser/expression/methods/match.rb +2 -0
- data/lib/regexp_parser/expression/methods/match_length.rb +2 -0
- data/lib/regexp_parser/expression/methods/negative.rb +22 -0
- data/lib/regexp_parser/expression/methods/options.rb +2 -0
- data/lib/regexp_parser/expression/methods/parts.rb +25 -0
- data/lib/regexp_parser/expression/methods/printing.rb +28 -0
- data/lib/regexp_parser/expression/methods/referenced_expressions.rb +30 -0
- data/lib/regexp_parser/expression/methods/strfregexp.rb +2 -0
- data/lib/regexp_parser/expression/methods/tests.rb +42 -3
- data/lib/regexp_parser/expression/methods/traverse.rb +35 -20
- data/lib/regexp_parser/expression/quantifier.rb +33 -18
- data/lib/regexp_parser/expression/sequence.rb +7 -9
- data/lib/regexp_parser/expression/sequence_operation.rb +6 -9
- data/lib/regexp_parser/expression/shared.rb +40 -24
- data/lib/regexp_parser/expression/subexpression.rb +22 -18
- data/lib/regexp_parser/expression.rb +39 -31
- data/lib/regexp_parser/lexer.rb +17 -7
- data/lib/regexp_parser/parser.rb +101 -95
- data/lib/regexp_parser/scanner/errors/premature_end_error.rb +10 -0
- data/lib/regexp_parser/scanner/errors/scanner_error.rb +8 -0
- data/lib/regexp_parser/scanner/errors/validation_error.rb +65 -0
- data/lib/regexp_parser/scanner/properties/long.csv +61 -0
- data/lib/regexp_parser/scanner/properties/short.csv +15 -0
- data/lib/regexp_parser/scanner/property.rl +1 -1
- data/lib/regexp_parser/scanner/scanner.rl +90 -140
- data/lib/regexp_parser/scanner.rb +1336 -1450
- data/lib/regexp_parser/syntax/any.rb +2 -0
- data/lib/regexp_parser/syntax/base.rb +2 -0
- data/lib/regexp_parser/syntax/token/anchor.rb +5 -3
- data/lib/regexp_parser/syntax/token/assertion.rb +4 -2
- data/lib/regexp_parser/syntax/token/backreference.rb +11 -6
- data/lib/regexp_parser/syntax/token/character_set.rb +6 -1
- data/lib/regexp_parser/syntax/token/character_type.rb +6 -4
- data/lib/regexp_parser/syntax/token/conditional.rb +5 -3
- data/lib/regexp_parser/syntax/token/escape.rb +12 -8
- data/lib/regexp_parser/syntax/token/group.rb +8 -6
- data/lib/regexp_parser/syntax/token/keep.rb +3 -1
- data/lib/regexp_parser/syntax/token/meta.rb +11 -2
- data/lib/regexp_parser/syntax/token/posix_class.rb +4 -2
- data/lib/regexp_parser/syntax/token/quantifier.rb +8 -6
- data/lib/regexp_parser/syntax/token/unicode_property.rb +164 -70
- data/lib/regexp_parser/syntax/token/virtual.rb +13 -0
- data/lib/regexp_parser/syntax/token.rb +18 -16
- data/lib/regexp_parser/syntax/version_lookup.rb +4 -2
- data/lib/regexp_parser/syntax/versions/1.8.6.rb +2 -0
- data/lib/regexp_parser/syntax/versions/1.9.1.rb +2 -0
- data/lib/regexp_parser/syntax/versions/1.9.3.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.0.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.2.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.3.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.4.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.4.1.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.5.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.6.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.6.2.rb +2 -0
- data/lib/regexp_parser/syntax/versions/2.6.3.rb +2 -0
- data/lib/regexp_parser/syntax/versions/3.1.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/3.2.0.rb +2 -0
- data/lib/regexp_parser/syntax/versions/3.5.0.rb +4 -0
- data/lib/regexp_parser/syntax/versions/4.0.0.rb +4 -0
- data/lib/regexp_parser/syntax/versions.rb +3 -1
- data/lib/regexp_parser/syntax.rb +3 -1
- data/lib/regexp_parser/token.rb +2 -0
- data/lib/regexp_parser/version.rb +3 -1
- data/lib/regexp_parser.rb +8 -6
- data/regexp_parser.gemspec +7 -5
- metadata +19 -11
- data/CHANGELOG.md +0 -632
- data/README.md +0 -503
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Regexp::Expression
|
|
2
4
|
class Subexpression < Regexp::Expression::Base
|
|
3
5
|
include Enumerable
|
|
@@ -11,16 +13,15 @@ module Regexp::Expression
|
|
|
11
13
|
|
|
12
14
|
# Override base method to clone the expressions as well.
|
|
13
15
|
def initialize_copy(orig)
|
|
14
|
-
self.expressions = orig.expressions.map
|
|
16
|
+
self.expressions = orig.expressions.map do |exp|
|
|
17
|
+
exp.clone.tap { |copy| copy.parent = self }
|
|
18
|
+
end
|
|
15
19
|
super
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
def <<(exp)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
else
|
|
22
|
-
expressions << exp
|
|
23
|
-
end
|
|
23
|
+
exp.parent = self
|
|
24
|
+
expressions << exp
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
%w[[] at each empty? fetch index join last length values_at].each do |method|
|
|
@@ -38,11 +39,7 @@ module Regexp::Expression
|
|
|
38
39
|
end
|
|
39
40
|
|
|
40
41
|
def te
|
|
41
|
-
ts +
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
def parts
|
|
45
|
-
expressions
|
|
42
|
+
ts + base_length
|
|
46
43
|
end
|
|
47
44
|
|
|
48
45
|
def to_h
|
|
@@ -52,14 +49,21 @@ module Regexp::Expression
|
|
|
52
49
|
)
|
|
53
50
|
end
|
|
54
51
|
|
|
55
|
-
def
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
52
|
+
def extract_quantifier_target(quantifier_description)
|
|
53
|
+
pre_quantifier_decorations = []
|
|
54
|
+
target = expressions.reverse.find do |exp|
|
|
55
|
+
if exp.decorative?
|
|
56
|
+
exp.custom_to_s_handling = true
|
|
57
|
+
pre_quantifier_decorations << exp.text
|
|
58
|
+
next
|
|
59
|
+
end
|
|
60
|
+
exp
|
|
61
|
+
end
|
|
62
|
+
target or raise Regexp::Parser::ParserError,
|
|
63
|
+
"No valid target found for '#{quantifier_description}' quantifier"
|
|
60
64
|
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
target.pre_quantifier_decorations = pre_quantifier_decorations
|
|
66
|
+
target
|
|
63
67
|
end
|
|
64
68
|
end
|
|
65
69
|
end
|
|
@@ -1,34 +1,42 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
require 'regexp_parser/expression/base'
|
|
5
|
-
require 'regexp_parser/expression/quantifier'
|
|
6
|
-
require 'regexp_parser/expression/subexpression'
|
|
7
|
-
require 'regexp_parser/expression/sequence'
|
|
8
|
-
require 'regexp_parser/expression/sequence_operation'
|
|
3
|
+
require_relative 'error'
|
|
9
4
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
require 'regexp_parser/expression/classes/character_type'
|
|
17
|
-
require 'regexp_parser/expression/classes/conditional'
|
|
18
|
-
require 'regexp_parser/expression/classes/escape_sequence'
|
|
19
|
-
require 'regexp_parser/expression/classes/free_space'
|
|
20
|
-
require 'regexp_parser/expression/classes/group'
|
|
21
|
-
require 'regexp_parser/expression/classes/keep'
|
|
22
|
-
require 'regexp_parser/expression/classes/literal'
|
|
23
|
-
require 'regexp_parser/expression/classes/posix_class'
|
|
24
|
-
require 'regexp_parser/expression/classes/root'
|
|
25
|
-
require 'regexp_parser/expression/classes/unicode_property'
|
|
5
|
+
require_relative 'expression/shared'
|
|
6
|
+
require_relative 'expression/base'
|
|
7
|
+
require_relative 'expression/quantifier'
|
|
8
|
+
require_relative 'expression/subexpression'
|
|
9
|
+
require_relative 'expression/sequence'
|
|
10
|
+
require_relative 'expression/sequence_operation'
|
|
26
11
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
12
|
+
require_relative 'expression/classes/alternation'
|
|
13
|
+
require_relative 'expression/classes/anchor'
|
|
14
|
+
require_relative 'expression/classes/backreference'
|
|
15
|
+
require_relative 'expression/classes/character_set'
|
|
16
|
+
require_relative 'expression/classes/character_set/intersection'
|
|
17
|
+
require_relative 'expression/classes/character_set/range'
|
|
18
|
+
require_relative 'expression/classes/character_type'
|
|
19
|
+
require_relative 'expression/classes/conditional'
|
|
20
|
+
require_relative 'expression/classes/escape_sequence'
|
|
21
|
+
require_relative 'expression/classes/free_space'
|
|
22
|
+
require_relative 'expression/classes/group'
|
|
23
|
+
require_relative 'expression/classes/keep'
|
|
24
|
+
require_relative 'expression/classes/literal'
|
|
25
|
+
require_relative 'expression/classes/posix_class'
|
|
26
|
+
require_relative 'expression/classes/root'
|
|
27
|
+
require_relative 'expression/classes/unicode_property'
|
|
28
|
+
|
|
29
|
+
require_relative 'expression/methods/construct'
|
|
30
|
+
require_relative 'expression/methods/escape_sequence_char'
|
|
31
|
+
require_relative 'expression/methods/escape_sequence_codepoint'
|
|
32
|
+
require_relative 'expression/methods/human_name'
|
|
33
|
+
require_relative 'expression/methods/match'
|
|
34
|
+
require_relative 'expression/methods/match_length'
|
|
35
|
+
require_relative 'expression/methods/negative'
|
|
36
|
+
require_relative 'expression/methods/options'
|
|
37
|
+
require_relative 'expression/methods/parts'
|
|
38
|
+
require_relative 'expression/methods/printing'
|
|
39
|
+
require_relative 'expression/methods/referenced_expressions'
|
|
40
|
+
require_relative 'expression/methods/strfregexp'
|
|
41
|
+
require_relative 'expression/methods/tests'
|
|
42
|
+
require_relative 'expression/methods/traverse'
|
data/lib/regexp_parser/lexer.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# A very thin wrapper around the scanner that breaks quantified literal runs,
|
|
2
4
|
# collects emitted tokens into an array, calculates their nesting depth, and
|
|
3
5
|
# normalizes tokens for the parser, and checks if they are implemented by the
|
|
@@ -6,7 +8,7 @@ class Regexp::Lexer
|
|
|
6
8
|
|
|
7
9
|
OPENING_TOKENS = %i[
|
|
8
10
|
capture passive lookahead nlookahead lookbehind nlookbehind
|
|
9
|
-
atomic options options_switch named absence
|
|
11
|
+
atomic options options_switch named absence open
|
|
10
12
|
].freeze
|
|
11
13
|
|
|
12
14
|
CLOSING_TOKENS = %i[close].freeze
|
|
@@ -89,24 +91,32 @@ class Regexp::Lexer
|
|
|
89
91
|
:nesting, :set_nesting, :conditional_nesting, :shift
|
|
90
92
|
|
|
91
93
|
def ascend(type, token)
|
|
94
|
+
return unless CLOSING_TOKENS.include?(token)
|
|
95
|
+
|
|
92
96
|
case type
|
|
93
97
|
when :group, :assertion
|
|
94
|
-
self.nesting = nesting - 1
|
|
98
|
+
self.nesting = nesting - 1
|
|
95
99
|
when :set
|
|
96
|
-
self.set_nesting = set_nesting - 1
|
|
100
|
+
self.set_nesting = set_nesting - 1
|
|
97
101
|
when :conditional
|
|
98
|
-
self.conditional_nesting = conditional_nesting - 1
|
|
102
|
+
self.conditional_nesting = conditional_nesting - 1
|
|
103
|
+
else
|
|
104
|
+
raise "unhandled nesting type #{type}"
|
|
99
105
|
end
|
|
100
106
|
end
|
|
101
107
|
|
|
102
108
|
def descend(type, token)
|
|
109
|
+
return unless OPENING_TOKENS.include?(token)
|
|
110
|
+
|
|
103
111
|
case type
|
|
104
112
|
when :group, :assertion
|
|
105
|
-
self.nesting = nesting + 1
|
|
113
|
+
self.nesting = nesting + 1
|
|
106
114
|
when :set
|
|
107
|
-
self.set_nesting = set_nesting + 1
|
|
115
|
+
self.set_nesting = set_nesting + 1
|
|
108
116
|
when :conditional
|
|
109
|
-
self.conditional_nesting = conditional_nesting + 1
|
|
117
|
+
self.conditional_nesting = conditional_nesting + 1
|
|
118
|
+
else
|
|
119
|
+
raise "unhandled nesting type #{type}"
|
|
110
120
|
end
|
|
111
121
|
end
|
|
112
122
|
|
data/lib/regexp_parser/parser.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'error'
|
|
4
|
+
require_relative 'expression'
|
|
3
5
|
|
|
4
6
|
class Regexp::Parser
|
|
5
7
|
include Regexp::Expression
|
|
@@ -232,7 +234,7 @@ class Regexp::Parser
|
|
|
232
234
|
node << Backreference::NameRecursionLevel.new(token, active_opts)
|
|
233
235
|
when :name_call
|
|
234
236
|
node << Backreference::NameCall.new(token, active_opts)
|
|
235
|
-
when :number, :number_ref
|
|
237
|
+
when :number, :number_ref # TODO: split in v3.0.0
|
|
236
238
|
node << Backreference::Number.new(token, active_opts)
|
|
237
239
|
when :number_recursion_ref
|
|
238
240
|
node << Backreference::NumberRecursionLevel.new(token, active_opts).tap do |exp|
|
|
@@ -272,9 +274,9 @@ class Regexp::Parser
|
|
|
272
274
|
nest_conditional(Conditional::Expression.new(token, active_opts))
|
|
273
275
|
when :condition
|
|
274
276
|
conditional_nesting.last.condition = Conditional::Condition.new(token, active_opts)
|
|
275
|
-
conditional_nesting.last.add_sequence(active_opts)
|
|
277
|
+
conditional_nesting.last.add_sequence(active_opts, { ts: token.te })
|
|
276
278
|
when :separator
|
|
277
|
-
conditional_nesting.last.add_sequence(active_opts)
|
|
279
|
+
conditional_nesting.last.add_sequence(active_opts, { ts: token.te })
|
|
278
280
|
self.node = conditional_nesting.last.branches.last
|
|
279
281
|
when :close
|
|
280
282
|
conditional_nesting.pop
|
|
@@ -319,9 +321,11 @@ class Regexp::Parser
|
|
|
319
321
|
when :codepoint_list; node << EscapeSequence::CodepointList.new(token, active_opts)
|
|
320
322
|
when :hex; node << EscapeSequence::Hex.new(token, active_opts)
|
|
321
323
|
when :octal; node << EscapeSequence::Octal.new(token, active_opts)
|
|
324
|
+
when :utf8_hex; node << EscapeSequence::UTF8Hex.new(token, active_opts)
|
|
322
325
|
|
|
323
326
|
when :control
|
|
324
327
|
if token.text =~ /\A(?:\\C-\\M|\\c\\M)/
|
|
328
|
+
# TODO: emit :meta_control_sequence token in v3.0.0
|
|
325
329
|
node << EscapeSequence::MetaControl.new(token, active_opts)
|
|
326
330
|
else
|
|
327
331
|
node << EscapeSequence::Control.new(token, active_opts)
|
|
@@ -329,6 +333,7 @@ class Regexp::Parser
|
|
|
329
333
|
|
|
330
334
|
when :meta_sequence
|
|
331
335
|
if token.text =~ /\A\\M-\\[Cc]/
|
|
336
|
+
# TODO: emit :meta_control_sequence token in v3.0.0:
|
|
332
337
|
node << EscapeSequence::MetaControl.new(token, active_opts)
|
|
333
338
|
else
|
|
334
339
|
node << EscapeSequence::Meta.new(token, active_opts)
|
|
@@ -349,11 +354,7 @@ class Regexp::Parser
|
|
|
349
354
|
when :comment
|
|
350
355
|
node << Comment.new(token, active_opts)
|
|
351
356
|
when :whitespace
|
|
352
|
-
|
|
353
|
-
node.last.merge(WhiteSpace.new(token, active_opts))
|
|
354
|
-
else
|
|
355
|
-
node << WhiteSpace.new(token, active_opts)
|
|
356
|
-
end
|
|
357
|
+
node << WhiteSpace.new(token, active_opts)
|
|
357
358
|
else
|
|
358
359
|
raise UnknownTokenError.new('FreeSpace', token)
|
|
359
360
|
end
|
|
@@ -381,96 +382,97 @@ class Regexp::Parser
|
|
|
381
382
|
def sequence_operation(klass, token)
|
|
382
383
|
unless node.instance_of?(klass)
|
|
383
384
|
operator = klass.new(token, active_opts)
|
|
384
|
-
sequence = operator.add_sequence(active_opts)
|
|
385
|
+
sequence = operator.add_sequence(active_opts, { ts: token.ts })
|
|
385
386
|
sequence.expressions = node.expressions
|
|
386
387
|
node.expressions = []
|
|
387
388
|
nest(operator)
|
|
388
389
|
end
|
|
389
|
-
node.add_sequence(active_opts)
|
|
390
|
+
node.add_sequence(active_opts, { ts: token.te })
|
|
390
391
|
end
|
|
391
392
|
|
|
392
393
|
def posixclass(token)
|
|
393
394
|
node << PosixClass.new(token, active_opts)
|
|
394
395
|
end
|
|
395
396
|
|
|
396
|
-
|
|
397
|
-
UPTokens = Regexp::Syntax::Token::
|
|
397
|
+
UP = Regexp::Expression::Property
|
|
398
|
+
UPTokens = Regexp::Syntax::Token::Property
|
|
398
399
|
|
|
399
400
|
def property(token)
|
|
400
401
|
case token.token
|
|
401
|
-
when :alnum; node << Alnum.new(token, active_opts)
|
|
402
|
-
when :alpha; node << Alpha.new(token, active_opts)
|
|
403
|
-
when :ascii; node << Ascii.new(token, active_opts)
|
|
404
|
-
when :blank; node << Blank.new(token, active_opts)
|
|
405
|
-
when :cntrl; node << Cntrl.new(token, active_opts)
|
|
406
|
-
when :digit; node << Digit.new(token, active_opts)
|
|
407
|
-
when :graph; node << Graph.new(token, active_opts)
|
|
408
|
-
when :lower; node << Lower.new(token, active_opts)
|
|
409
|
-
when :print; node << Print.new(token, active_opts)
|
|
410
|
-
when :punct; node << Punct.new(token, active_opts)
|
|
411
|
-
when :space; node << Space.new(token, active_opts)
|
|
412
|
-
when :upper; node << Upper.new(token, active_opts)
|
|
413
|
-
when :word; node << Word.new(token, active_opts)
|
|
414
|
-
when :xdigit; node << Xdigit.new(token, active_opts)
|
|
415
|
-
when :xposixpunct; node << XPosixPunct.new(token, active_opts)
|
|
402
|
+
when :alnum; node << UP::Alnum.new(token, active_opts)
|
|
403
|
+
when :alpha; node << UP::Alpha.new(token, active_opts)
|
|
404
|
+
when :ascii; node << UP::Ascii.new(token, active_opts)
|
|
405
|
+
when :blank; node << UP::Blank.new(token, active_opts)
|
|
406
|
+
when :cntrl; node << UP::Cntrl.new(token, active_opts)
|
|
407
|
+
when :digit; node << UP::Digit.new(token, active_opts)
|
|
408
|
+
when :graph; node << UP::Graph.new(token, active_opts)
|
|
409
|
+
when :lower; node << UP::Lower.new(token, active_opts)
|
|
410
|
+
when :print; node << UP::Print.new(token, active_opts)
|
|
411
|
+
when :punct; node << UP::Punct.new(token, active_opts)
|
|
412
|
+
when :space; node << UP::Space.new(token, active_opts)
|
|
413
|
+
when :upper; node << UP::Upper.new(token, active_opts)
|
|
414
|
+
when :word; node << UP::Word.new(token, active_opts)
|
|
415
|
+
when :xdigit; node << UP::Xdigit.new(token, active_opts)
|
|
416
|
+
when :xposixpunct; node << UP::XPosixPunct.new(token, active_opts)
|
|
416
417
|
|
|
417
418
|
# only in Oniguruma (old rubies)
|
|
418
|
-
when :newline; node << Newline.new(token, active_opts)
|
|
419
|
-
|
|
420
|
-
when :any; node << Any.new(token, active_opts)
|
|
421
|
-
when :assigned; node << Assigned.new(token, active_opts)
|
|
422
|
-
|
|
423
|
-
when :letter; node << Letter::Any.new(token, active_opts)
|
|
424
|
-
when :cased_letter; node << Letter::Cased.new(token, active_opts)
|
|
425
|
-
when :uppercase_letter; node << Letter::Uppercase.new(token, active_opts)
|
|
426
|
-
when :lowercase_letter; node << Letter::Lowercase.new(token, active_opts)
|
|
427
|
-
when :titlecase_letter; node << Letter::Titlecase.new(token, active_opts)
|
|
428
|
-
when :modifier_letter; node << Letter::Modifier.new(token, active_opts)
|
|
429
|
-
when :other_letter; node << Letter::Other.new(token, active_opts)
|
|
430
|
-
|
|
431
|
-
when :mark; node << Mark::Any.new(token, active_opts)
|
|
432
|
-
when :combining_mark; node << Mark::Combining.new(token, active_opts)
|
|
433
|
-
when :nonspacing_mark; node << Mark::Nonspacing.new(token, active_opts)
|
|
434
|
-
when :spacing_mark; node << Mark::Spacing.new(token, active_opts)
|
|
435
|
-
when :enclosing_mark; node << Mark::Enclosing.new(token, active_opts)
|
|
436
|
-
|
|
437
|
-
when :number; node << Number::Any.new(token, active_opts)
|
|
438
|
-
when :decimal_number; node << Number::Decimal.new(token, active_opts)
|
|
439
|
-
when :letter_number; node << Number::Letter.new(token, active_opts)
|
|
440
|
-
when :other_number; node << Number::Other.new(token, active_opts)
|
|
441
|
-
|
|
442
|
-
when :punctuation; node << Punctuation::Any.new(token, active_opts)
|
|
443
|
-
when :connector_punctuation; node << Punctuation::Connector.new(token, active_opts)
|
|
444
|
-
when :dash_punctuation; node << Punctuation::Dash.new(token, active_opts)
|
|
445
|
-
when :open_punctuation; node << Punctuation::Open.new(token, active_opts)
|
|
446
|
-
when :close_punctuation; node << Punctuation::Close.new(token, active_opts)
|
|
447
|
-
when :initial_punctuation; node << Punctuation::Initial.new(token, active_opts)
|
|
448
|
-
when :final_punctuation; node << Punctuation::Final.new(token, active_opts)
|
|
449
|
-
when :other_punctuation; node << Punctuation::Other.new(token, active_opts)
|
|
450
|
-
|
|
451
|
-
when :separator; node << Separator::Any.new(token, active_opts)
|
|
452
|
-
when :space_separator; node << Separator::Space.new(token, active_opts)
|
|
453
|
-
when :line_separator; node << Separator::Line.new(token, active_opts)
|
|
454
|
-
when :paragraph_separator; node << Separator::Paragraph.new(token, active_opts)
|
|
455
|
-
|
|
456
|
-
when :symbol; node << Symbol::Any.new(token, active_opts)
|
|
457
|
-
when :math_symbol; node << Symbol::Math.new(token, active_opts)
|
|
458
|
-
when :currency_symbol; node << Symbol::Currency.new(token, active_opts)
|
|
459
|
-
when :modifier_symbol; node << Symbol::Modifier.new(token, active_opts)
|
|
460
|
-
when :other_symbol; node << Symbol::Other.new(token, active_opts)
|
|
461
|
-
|
|
462
|
-
when :other; node << Codepoint::Any.new(token, active_opts)
|
|
463
|
-
when :control; node << Codepoint::Control.new(token, active_opts)
|
|
464
|
-
when :format; node << Codepoint::Format.new(token, active_opts)
|
|
465
|
-
when :surrogate; node << Codepoint::Surrogate.new(token, active_opts)
|
|
466
|
-
when :private_use; node << Codepoint::PrivateUse.new(token, active_opts)
|
|
467
|
-
when :unassigned; node << Codepoint::Unassigned.new(token, active_opts)
|
|
468
|
-
|
|
469
|
-
when *UPTokens::Age; node << Age.new(token, active_opts)
|
|
470
|
-
when *UPTokens::Derived; node << Derived.new(token, active_opts)
|
|
471
|
-
when *UPTokens::Emoji; node << Emoji.new(token, active_opts)
|
|
472
|
-
when *UPTokens::
|
|
473
|
-
when *UPTokens::
|
|
419
|
+
when :newline; node << UP::Newline.new(token, active_opts)
|
|
420
|
+
|
|
421
|
+
when :any; node << UP::Any.new(token, active_opts)
|
|
422
|
+
when :assigned; node << UP::Assigned.new(token, active_opts)
|
|
423
|
+
|
|
424
|
+
when :letter; node << UP::Letter::Any.new(token, active_opts)
|
|
425
|
+
when :cased_letter; node << UP::Letter::Cased.new(token, active_opts)
|
|
426
|
+
when :uppercase_letter; node << UP::Letter::Uppercase.new(token, active_opts)
|
|
427
|
+
when :lowercase_letter; node << UP::Letter::Lowercase.new(token, active_opts)
|
|
428
|
+
when :titlecase_letter; node << UP::Letter::Titlecase.new(token, active_opts)
|
|
429
|
+
when :modifier_letter; node << UP::Letter::Modifier.new(token, active_opts)
|
|
430
|
+
when :other_letter; node << UP::Letter::Other.new(token, active_opts)
|
|
431
|
+
|
|
432
|
+
when :mark; node << UP::Mark::Any.new(token, active_opts)
|
|
433
|
+
when :combining_mark; node << UP::Mark::Combining.new(token, active_opts)
|
|
434
|
+
when :nonspacing_mark; node << UP::Mark::Nonspacing.new(token, active_opts)
|
|
435
|
+
when :spacing_mark; node << UP::Mark::Spacing.new(token, active_opts)
|
|
436
|
+
when :enclosing_mark; node << UP::Mark::Enclosing.new(token, active_opts)
|
|
437
|
+
|
|
438
|
+
when :number; node << UP::Number::Any.new(token, active_opts)
|
|
439
|
+
when :decimal_number; node << UP::Number::Decimal.new(token, active_opts)
|
|
440
|
+
when :letter_number; node << UP::Number::Letter.new(token, active_opts)
|
|
441
|
+
when :other_number; node << UP::Number::Other.new(token, active_opts)
|
|
442
|
+
|
|
443
|
+
when :punctuation; node << UP::Punctuation::Any.new(token, active_opts)
|
|
444
|
+
when :connector_punctuation; node << UP::Punctuation::Connector.new(token, active_opts)
|
|
445
|
+
when :dash_punctuation; node << UP::Punctuation::Dash.new(token, active_opts)
|
|
446
|
+
when :open_punctuation; node << UP::Punctuation::Open.new(token, active_opts)
|
|
447
|
+
when :close_punctuation; node << UP::Punctuation::Close.new(token, active_opts)
|
|
448
|
+
when :initial_punctuation; node << UP::Punctuation::Initial.new(token, active_opts)
|
|
449
|
+
when :final_punctuation; node << UP::Punctuation::Final.new(token, active_opts)
|
|
450
|
+
when :other_punctuation; node << UP::Punctuation::Other.new(token, active_opts)
|
|
451
|
+
|
|
452
|
+
when :separator; node << UP::Separator::Any.new(token, active_opts)
|
|
453
|
+
when :space_separator; node << UP::Separator::Space.new(token, active_opts)
|
|
454
|
+
when :line_separator; node << UP::Separator::Line.new(token, active_opts)
|
|
455
|
+
when :paragraph_separator; node << UP::Separator::Paragraph.new(token, active_opts)
|
|
456
|
+
|
|
457
|
+
when :symbol; node << UP::Symbol::Any.new(token, active_opts)
|
|
458
|
+
when :math_symbol; node << UP::Symbol::Math.new(token, active_opts)
|
|
459
|
+
when :currency_symbol; node << UP::Symbol::Currency.new(token, active_opts)
|
|
460
|
+
when :modifier_symbol; node << UP::Symbol::Modifier.new(token, active_opts)
|
|
461
|
+
when :other_symbol; node << UP::Symbol::Other.new(token, active_opts)
|
|
462
|
+
|
|
463
|
+
when :other; node << UP::Codepoint::Any.new(token, active_opts)
|
|
464
|
+
when :control; node << UP::Codepoint::Control.new(token, active_opts)
|
|
465
|
+
when :format; node << UP::Codepoint::Format.new(token, active_opts)
|
|
466
|
+
when :surrogate; node << UP::Codepoint::Surrogate.new(token, active_opts)
|
|
467
|
+
when :private_use; node << UP::Codepoint::PrivateUse.new(token, active_opts)
|
|
468
|
+
when :unassigned; node << UP::Codepoint::Unassigned.new(token, active_opts)
|
|
469
|
+
|
|
470
|
+
when *UPTokens::Age; node << UP::Age.new(token, active_opts)
|
|
471
|
+
when *UPTokens::Derived; node << UP::Derived.new(token, active_opts)
|
|
472
|
+
when *UPTokens::Emoji; node << UP::Emoji.new(token, active_opts)
|
|
473
|
+
when *UPTokens::Enumerated; node << UP::Enumerated.new(token, active_opts)
|
|
474
|
+
when *UPTokens::Script; node << UP::Script.new(token, active_opts)
|
|
475
|
+
when *UPTokens::UnicodeBlock; node << UP::Block.new(token, active_opts)
|
|
474
476
|
|
|
475
477
|
else
|
|
476
478
|
raise UnknownTokenError.new('UnicodeProperty', token)
|
|
@@ -478,8 +480,7 @@ class Regexp::Parser
|
|
|
478
480
|
end
|
|
479
481
|
|
|
480
482
|
def quantifier(token)
|
|
481
|
-
target_node = node.
|
|
482
|
-
target_node or raise ParserError, "No valid target found for '#{token.text}'"
|
|
483
|
+
target_node = node.extract_quantifier_target(token.text)
|
|
483
484
|
|
|
484
485
|
# in case of chained quantifiers, wrap target in an implicit passive group
|
|
485
486
|
# description of the problem: https://github.com/ammar/regexp_parser/issues/3
|
|
@@ -527,6 +528,8 @@ class Regexp::Parser
|
|
|
527
528
|
end
|
|
528
529
|
|
|
529
530
|
def open_set(token)
|
|
531
|
+
# TODO: this and Quantifier are the only cases where Expression#token
|
|
532
|
+
# does not match the scanner/lexer output. Fix in v3.0.0.
|
|
530
533
|
token.token = :character
|
|
531
534
|
nest(CharacterSet.new(token, active_opts))
|
|
532
535
|
end
|
|
@@ -575,22 +578,25 @@ class Regexp::Parser
|
|
|
575
578
|
options_stack.last
|
|
576
579
|
end
|
|
577
580
|
|
|
578
|
-
# Assigns referenced expressions to
|
|
581
|
+
# Assigns referenced expressions to referring expressions, e.g. if there is
|
|
579
582
|
# an instance of Backreference::Number, its #referenced_expression is set to
|
|
580
583
|
# the instance of Group::Capture that it refers to via its number.
|
|
581
584
|
def assign_referenced_expressions
|
|
582
|
-
# find all
|
|
583
|
-
targets = { 0 => root }
|
|
585
|
+
# find all referenceable and referring expressions
|
|
586
|
+
targets = { 0 => [root] }
|
|
584
587
|
referrers = []
|
|
585
588
|
root.each_expression do |exp|
|
|
586
|
-
exp.
|
|
587
|
-
|
|
589
|
+
if exp.referential?
|
|
590
|
+
referrers << exp
|
|
591
|
+
elsif exp.is_a?(Group::Capture)
|
|
592
|
+
(targets[exp.identifier] ||= []) << exp
|
|
593
|
+
end
|
|
588
594
|
end
|
|
589
|
-
# assign
|
|
595
|
+
# assign referenced expressions to referring expressions
|
|
590
596
|
# (in a second iteration because there might be forward references)
|
|
591
597
|
referrers.each do |exp|
|
|
592
|
-
exp.
|
|
593
|
-
raise(ParserError, "Invalid reference
|
|
598
|
+
exp.referenced_expressions = targets[exp.reference] ||
|
|
599
|
+
raise(ParserError, "Invalid reference #{exp.reference} at pos #{exp.ts}")
|
|
594
600
|
end
|
|
595
601
|
end
|
|
596
602
|
end # module Regexp::Parser
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class Regexp::Scanner
|
|
4
|
+
# Base for all scanner validation errors
|
|
5
|
+
class ValidationError < ScannerError
|
|
6
|
+
# Centralizes and unifies the handling of validation related errors.
|
|
7
|
+
def self.for(type, problem, reason = nil)
|
|
8
|
+
types.fetch(type).new(problem, reason)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.types
|
|
12
|
+
@types ||= {
|
|
13
|
+
backref: InvalidBackrefError,
|
|
14
|
+
group: InvalidGroupError,
|
|
15
|
+
group_option: InvalidGroupOption,
|
|
16
|
+
posix_class: UnknownPosixClassError,
|
|
17
|
+
property: UnknownUnicodePropertyError,
|
|
18
|
+
sequence: InvalidSequenceError,
|
|
19
|
+
}
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Invalid sequence format. Used for escape sequences, mainly.
|
|
24
|
+
class InvalidSequenceError < ValidationError
|
|
25
|
+
def initialize(what = 'sequence', where = '')
|
|
26
|
+
super "Invalid #{what} at #{where}"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Invalid group. Used for named groups.
|
|
31
|
+
class InvalidGroupError < ValidationError
|
|
32
|
+
def initialize(what, reason)
|
|
33
|
+
super "Invalid #{what}, #{reason}."
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Invalid groupOption. Used for inline options.
|
|
38
|
+
# TODO: should become InvalidGroupOptionError in v3.0.0 for consistency
|
|
39
|
+
class InvalidGroupOption < ValidationError
|
|
40
|
+
def initialize(option, text)
|
|
41
|
+
super "Invalid group option #{option} in #{text}"
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Invalid back reference. Used for name a number refs/calls.
|
|
46
|
+
class InvalidBackrefError < ValidationError
|
|
47
|
+
def initialize(what, reason)
|
|
48
|
+
super "Invalid back reference #{what}, #{reason}"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The property name was not recognized by the scanner.
|
|
53
|
+
class UnknownUnicodePropertyError < ValidationError
|
|
54
|
+
def initialize(name, _)
|
|
55
|
+
super "Unknown unicode character property name #{name}"
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The POSIX class name was not recognized by the scanner.
|
|
60
|
+
class UnknownPosixClassError < ValidationError
|
|
61
|
+
def initialize(text, _)
|
|
62
|
+
super "Unknown POSIX class #{text}"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|