regexp_parser 2.1.1 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/Gemfile +1 -1
- data/LICENSE +1 -1
- data/README.md +31 -27
- data/Rakefile +6 -70
- data/lib/regexp_parser/expression/base.rb +123 -0
- data/lib/regexp_parser/expression/classes/anchor.rb +0 -2
- data/lib/regexp_parser/expression/classes/{backref.rb → backreference.rb} +0 -0
- data/lib/regexp_parser/expression/classes/{set → character_set}/intersection.rb +0 -0
- data/lib/regexp_parser/expression/classes/{set → character_set}/range.rb +0 -0
- data/lib/regexp_parser/expression/classes/{set.rb → character_set.rb} +0 -0
- data/lib/regexp_parser/expression/classes/{escape.rb → escape_sequence.rb} +13 -7
- data/lib/regexp_parser/expression/classes/free_space.rb +0 -2
- data/lib/regexp_parser/expression/classes/literal.rb +1 -5
- data/lib/regexp_parser/expression/classes/property.rb +0 -2
- data/lib/regexp_parser/expression/classes/root.rb +0 -1
- data/lib/regexp_parser/expression/classes/type.rb +0 -2
- data/lib/regexp_parser/expression/methods/strfregexp.rb +1 -1
- data/lib/regexp_parser/expression/quantifier.rb +1 -1
- data/lib/regexp_parser/expression/sequence.rb +0 -1
- data/lib/regexp_parser/expression/subexpression.rb +0 -1
- data/lib/regexp_parser/expression.rb +6 -130
- data/lib/regexp_parser/lexer.rb +8 -6
- data/lib/regexp_parser/scanner/properties/long.csv +622 -0
- data/lib/regexp_parser/scanner/properties/short.csv +246 -0
- data/lib/regexp_parser/scanner/scanner.rl +6 -4
- data/lib/regexp_parser/scanner.rb +126 -124
- data/lib/regexp_parser/syntax/any.rb +2 -7
- data/lib/regexp_parser/syntax/base.rb +91 -66
- data/lib/regexp_parser/syntax/token/anchor.rb +15 -0
- data/lib/regexp_parser/syntax/{tokens → token}/assertion.rb +2 -2
- data/lib/regexp_parser/syntax/token/backreference.rb +30 -0
- data/lib/regexp_parser/syntax/{tokens → token}/character_set.rb +2 -2
- data/lib/regexp_parser/syntax/{tokens → token}/character_type.rb +3 -3
- data/lib/regexp_parser/syntax/{tokens → token}/conditional.rb +3 -3
- data/lib/regexp_parser/syntax/token/escape.rb +31 -0
- data/lib/regexp_parser/syntax/{tokens → token}/group.rb +7 -7
- data/lib/regexp_parser/syntax/{tokens → token}/keep.rb +1 -1
- data/lib/regexp_parser/syntax/{tokens → token}/meta.rb +2 -2
- data/lib/regexp_parser/syntax/{tokens → token}/posix_class.rb +3 -3
- data/lib/regexp_parser/syntax/token/quantifier.rb +35 -0
- data/lib/regexp_parser/syntax/token/unicode_property.rb +722 -0
- data/lib/regexp_parser/syntax/token.rb +45 -0
- data/lib/regexp_parser/syntax/version_lookup.rb +20 -29
- data/lib/regexp_parser/syntax/versions/1.8.6.rb +13 -20
- data/lib/regexp_parser/syntax/versions/1.9.1.rb +10 -17
- data/lib/regexp_parser/syntax/versions/1.9.3.rb +3 -10
- data/lib/regexp_parser/syntax/versions/2.0.0.rb +8 -15
- data/lib/regexp_parser/syntax/versions/2.2.0.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.3.0.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.4.0.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.4.1.rb +2 -8
- data/lib/regexp_parser/syntax/versions/2.5.0.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.6.0.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.6.2.rb +3 -9
- data/lib/regexp_parser/syntax/versions/2.6.3.rb +3 -9
- data/lib/regexp_parser/syntax/versions/3.1.0.rb +4 -0
- data/lib/regexp_parser/syntax/versions/3.2.0.rb +4 -0
- data/lib/regexp_parser/syntax/versions.rb +1 -1
- data/lib/regexp_parser/syntax.rb +1 -1
- data/lib/regexp_parser/token.rb +9 -20
- data/lib/regexp_parser/version.rb +1 -1
- data/lib/regexp_parser.rb +0 -2
- data/regexp_parser.gemspec +20 -22
- metadata +36 -167
- data/lib/regexp_parser/scanner/properties/long.yml +0 -594
- data/lib/regexp_parser/scanner/properties/short.yml +0 -237
- data/lib/regexp_parser/syntax/tokens/anchor.rb +0 -15
- data/lib/regexp_parser/syntax/tokens/backref.rb +0 -24
- data/lib/regexp_parser/syntax/tokens/escape.rb +0 -30
- data/lib/regexp_parser/syntax/tokens/quantifier.rb +0 -35
- data/lib/regexp_parser/syntax/tokens/unicode_property.rb +0 -675
- data/lib/regexp_parser/syntax/tokens.rb +0 -45
- data/spec/expression/base_spec.rb +0 -104
- data/spec/expression/clone_spec.rb +0 -152
- data/spec/expression/conditional_spec.rb +0 -89
- data/spec/expression/free_space_spec.rb +0 -27
- data/spec/expression/methods/match_length_spec.rb +0 -161
- data/spec/expression/methods/match_spec.rb +0 -25
- data/spec/expression/methods/strfregexp_spec.rb +0 -224
- data/spec/expression/methods/tests_spec.rb +0 -99
- data/spec/expression/methods/traverse_spec.rb +0 -161
- data/spec/expression/options_spec.rb +0 -128
- data/spec/expression/subexpression_spec.rb +0 -50
- data/spec/expression/to_h_spec.rb +0 -26
- data/spec/expression/to_s_spec.rb +0 -108
- data/spec/lexer/all_spec.rb +0 -22
- data/spec/lexer/conditionals_spec.rb +0 -53
- data/spec/lexer/delimiters_spec.rb +0 -68
- data/spec/lexer/escapes_spec.rb +0 -14
- data/spec/lexer/keep_spec.rb +0 -10
- data/spec/lexer/literals_spec.rb +0 -64
- data/spec/lexer/nesting_spec.rb +0 -99
- data/spec/lexer/refcalls_spec.rb +0 -60
- data/spec/parser/all_spec.rb +0 -43
- data/spec/parser/alternation_spec.rb +0 -88
- data/spec/parser/anchors_spec.rb +0 -17
- data/spec/parser/conditionals_spec.rb +0 -179
- data/spec/parser/errors_spec.rb +0 -30
- data/spec/parser/escapes_spec.rb +0 -121
- data/spec/parser/free_space_spec.rb +0 -130
- data/spec/parser/groups_spec.rb +0 -108
- data/spec/parser/keep_spec.rb +0 -6
- data/spec/parser/options_spec.rb +0 -28
- data/spec/parser/posix_classes_spec.rb +0 -8
- data/spec/parser/properties_spec.rb +0 -115
- data/spec/parser/quantifiers_spec.rb +0 -68
- data/spec/parser/refcalls_spec.rb +0 -117
- data/spec/parser/set/intersections_spec.rb +0 -127
- data/spec/parser/set/ranges_spec.rb +0 -111
- data/spec/parser/sets_spec.rb +0 -178
- data/spec/parser/types_spec.rb +0 -18
- data/spec/scanner/all_spec.rb +0 -18
- data/spec/scanner/anchors_spec.rb +0 -21
- data/spec/scanner/conditionals_spec.rb +0 -128
- data/spec/scanner/delimiters_spec.rb +0 -52
- data/spec/scanner/errors_spec.rb +0 -67
- data/spec/scanner/escapes_spec.rb +0 -64
- data/spec/scanner/free_space_spec.rb +0 -165
- data/spec/scanner/groups_spec.rb +0 -61
- data/spec/scanner/keep_spec.rb +0 -10
- data/spec/scanner/literals_spec.rb +0 -39
- data/spec/scanner/meta_spec.rb +0 -18
- data/spec/scanner/options_spec.rb +0 -36
- data/spec/scanner/properties_spec.rb +0 -64
- data/spec/scanner/quantifiers_spec.rb +0 -25
- data/spec/scanner/refcalls_spec.rb +0 -55
- data/spec/scanner/sets_spec.rb +0 -151
- data/spec/scanner/types_spec.rb +0 -14
- data/spec/spec_helper.rb +0 -16
- data/spec/support/runner.rb +0 -42
- data/spec/support/shared_examples.rb +0 -77
- data/spec/support/warning_extractor.rb +0 -60
- data/spec/syntax/syntax_spec.rb +0 -48
- data/spec/syntax/syntax_token_map_spec.rb +0 -23
- data/spec/syntax/versions/1.8.6_spec.rb +0 -17
- data/spec/syntax/versions/1.9.1_spec.rb +0 -10
- data/spec/syntax/versions/1.9.3_spec.rb +0 -9
- data/spec/syntax/versions/2.0.0_spec.rb +0 -13
- data/spec/syntax/versions/2.2.0_spec.rb +0 -9
- data/spec/syntax/versions/aliases_spec.rb +0 -37
- data/spec/token/token_spec.rb +0 -85
@@ -1,130 +1,6 @@
|
|
1
1
|
require 'regexp_parser/error'
|
2
2
|
|
3
|
-
|
4
|
-
class Base
|
5
|
-
attr_accessor :type, :token
|
6
|
-
attr_accessor :text, :ts
|
7
|
-
attr_accessor :level, :set_level, :conditional_level, :nesting_level
|
8
|
-
|
9
|
-
attr_accessor :quantifier
|
10
|
-
attr_accessor :options
|
11
|
-
|
12
|
-
def initialize(token, options = {})
|
13
|
-
self.type = token.type
|
14
|
-
self.token = token.token
|
15
|
-
self.text = token.text
|
16
|
-
self.ts = token.ts
|
17
|
-
self.level = token.level
|
18
|
-
self.set_level = token.set_level
|
19
|
-
self.conditional_level = token.conditional_level
|
20
|
-
self.nesting_level = 0
|
21
|
-
self.quantifier = nil
|
22
|
-
self.options = options
|
23
|
-
end
|
24
|
-
|
25
|
-
def initialize_copy(orig)
|
26
|
-
self.text = (orig.text ? orig.text.dup : nil)
|
27
|
-
self.options = (orig.options ? orig.options.dup : nil)
|
28
|
-
self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
|
29
|
-
super
|
30
|
-
end
|
31
|
-
|
32
|
-
def to_re(format = :full)
|
33
|
-
::Regexp.new(to_s(format))
|
34
|
-
end
|
35
|
-
|
36
|
-
alias :starts_at :ts
|
37
|
-
|
38
|
-
def base_length
|
39
|
-
to_s(:base).length
|
40
|
-
end
|
41
|
-
|
42
|
-
def full_length
|
43
|
-
to_s.length
|
44
|
-
end
|
45
|
-
|
46
|
-
def offset
|
47
|
-
[starts_at, full_length]
|
48
|
-
end
|
49
|
-
|
50
|
-
def coded_offset
|
51
|
-
'@%d+%d' % offset
|
52
|
-
end
|
53
|
-
|
54
|
-
def to_s(format = :full)
|
55
|
-
"#{text}#{quantifier_affix(format)}"
|
56
|
-
end
|
57
|
-
|
58
|
-
def quantifier_affix(expression_format)
|
59
|
-
quantifier.to_s if quantified? && expression_format != :base
|
60
|
-
end
|
61
|
-
|
62
|
-
def terminal?
|
63
|
-
!respond_to?(:expressions)
|
64
|
-
end
|
65
|
-
|
66
|
-
def quantify(token, text, min = nil, max = nil, mode = :greedy)
|
67
|
-
self.quantifier = Quantifier.new(token, text, min, max, mode)
|
68
|
-
end
|
69
|
-
|
70
|
-
def unquantified_clone
|
71
|
-
clone.tap { |exp| exp.quantifier = nil }
|
72
|
-
end
|
73
|
-
|
74
|
-
def quantified?
|
75
|
-
!quantifier.nil?
|
76
|
-
end
|
77
|
-
|
78
|
-
# Deprecated. Prefer `#repetitions` which has a more uniform interface.
|
79
|
-
def quantity
|
80
|
-
return [nil,nil] unless quantified?
|
81
|
-
[quantifier.min, quantifier.max]
|
82
|
-
end
|
83
|
-
|
84
|
-
def repetitions
|
85
|
-
return 1..1 unless quantified?
|
86
|
-
min = quantifier.min
|
87
|
-
max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
|
88
|
-
range = min..max
|
89
|
-
# fix Range#minmax on old Rubies - https://bugs.ruby-lang.org/issues/15807
|
90
|
-
if RUBY_VERSION.to_f < 2.7
|
91
|
-
range.define_singleton_method(:minmax) { [min, max] }
|
92
|
-
end
|
93
|
-
range
|
94
|
-
end
|
95
|
-
|
96
|
-
def greedy?
|
97
|
-
quantified? and quantifier.greedy?
|
98
|
-
end
|
99
|
-
|
100
|
-
def reluctant?
|
101
|
-
quantified? and quantifier.reluctant?
|
102
|
-
end
|
103
|
-
alias :lazy? :reluctant?
|
104
|
-
|
105
|
-
def possessive?
|
106
|
-
quantified? and quantifier.possessive?
|
107
|
-
end
|
108
|
-
|
109
|
-
def attributes
|
110
|
-
{
|
111
|
-
type: type,
|
112
|
-
token: token,
|
113
|
-
text: to_s(:base),
|
114
|
-
starts_at: ts,
|
115
|
-
length: full_length,
|
116
|
-
level: level,
|
117
|
-
set_level: set_level,
|
118
|
-
conditional_level: conditional_level,
|
119
|
-
options: options,
|
120
|
-
quantifier: quantified? ? quantifier.to_h : nil,
|
121
|
-
}
|
122
|
-
end
|
123
|
-
alias :to_h :attributes
|
124
|
-
end
|
125
|
-
|
126
|
-
end # module Regexp::Expression
|
127
|
-
|
3
|
+
require 'regexp_parser/expression/base'
|
128
4
|
require 'regexp_parser/expression/quantifier'
|
129
5
|
require 'regexp_parser/expression/subexpression'
|
130
6
|
require 'regexp_parser/expression/sequence'
|
@@ -132,9 +8,12 @@ require 'regexp_parser/expression/sequence_operation'
|
|
132
8
|
|
133
9
|
require 'regexp_parser/expression/classes/alternation'
|
134
10
|
require 'regexp_parser/expression/classes/anchor'
|
135
|
-
require 'regexp_parser/expression/classes/
|
11
|
+
require 'regexp_parser/expression/classes/backreference'
|
12
|
+
require 'regexp_parser/expression/classes/character_set'
|
13
|
+
require 'regexp_parser/expression/classes/character_set/intersection'
|
14
|
+
require 'regexp_parser/expression/classes/character_set/range'
|
136
15
|
require 'regexp_parser/expression/classes/conditional'
|
137
|
-
require 'regexp_parser/expression/classes/
|
16
|
+
require 'regexp_parser/expression/classes/escape_sequence'
|
138
17
|
require 'regexp_parser/expression/classes/free_space'
|
139
18
|
require 'regexp_parser/expression/classes/group'
|
140
19
|
require 'regexp_parser/expression/classes/keep'
|
@@ -142,9 +21,6 @@ require 'regexp_parser/expression/classes/literal'
|
|
142
21
|
require 'regexp_parser/expression/classes/posix_class'
|
143
22
|
require 'regexp_parser/expression/classes/property'
|
144
23
|
require 'regexp_parser/expression/classes/root'
|
145
|
-
require 'regexp_parser/expression/classes/set'
|
146
|
-
require 'regexp_parser/expression/classes/set/intersection'
|
147
|
-
require 'regexp_parser/expression/classes/set/range'
|
148
24
|
require 'regexp_parser/expression/classes/type'
|
149
25
|
|
150
26
|
require 'regexp_parser/expression/methods/match'
|
data/lib/regexp_parser/lexer.rb
CHANGED
@@ -4,19 +4,21 @@
|
|
4
4
|
# given syntax flavor.
|
5
5
|
class Regexp::Lexer
|
6
6
|
|
7
|
-
OPENING_TOKENS = [
|
8
|
-
|
9
|
-
|
7
|
+
OPENING_TOKENS = %i[
|
8
|
+
capture passive lookahead nlookahead lookbehind nlookbehind
|
9
|
+
atomic options options_switch named absence
|
10
10
|
].freeze
|
11
11
|
|
12
|
-
CLOSING_TOKENS = [
|
12
|
+
CLOSING_TOKENS = %i[close].freeze
|
13
|
+
|
14
|
+
CONDITION_TOKENS = %i[condition condition_close].freeze
|
13
15
|
|
14
16
|
def self.lex(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
|
15
17
|
new.lex(input, syntax, options: options, &block)
|
16
18
|
end
|
17
19
|
|
18
20
|
def lex(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
|
19
|
-
syntax = Regexp::Syntax.
|
21
|
+
syntax = Regexp::Syntax.for(syntax)
|
20
22
|
|
21
23
|
self.tokens = []
|
22
24
|
self.nesting = 0
|
@@ -40,7 +42,7 @@ class Regexp::Lexer
|
|
40
42
|
nesting, set_nesting, conditional_nesting)
|
41
43
|
|
42
44
|
current = merge_condition(current) if type == :conditional and
|
43
|
-
|
45
|
+
CONDITION_TOKENS.include?(token)
|
44
46
|
|
45
47
|
last.next = current if last
|
46
48
|
current.previous = last if last
|