regexp_parser 1.3.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -1
  3. data/Gemfile +3 -3
  4. data/README.md +12 -19
  5. data/Rakefile +3 -4
  6. data/lib/regexp_parser/expression.rb +28 -53
  7. data/lib/regexp_parser/expression/classes/backref.rb +18 -10
  8. data/lib/regexp_parser/expression/classes/conditional.rb +7 -2
  9. data/lib/regexp_parser/expression/classes/escape.rb +0 -4
  10. data/lib/regexp_parser/expression/classes/group.rb +4 -2
  11. data/lib/regexp_parser/expression/classes/keep.rb +1 -3
  12. data/lib/regexp_parser/expression/methods/match.rb +13 -0
  13. data/lib/regexp_parser/expression/methods/match_length.rb +172 -0
  14. data/lib/regexp_parser/expression/methods/options.rb +35 -0
  15. data/lib/regexp_parser/expression/methods/strfregexp.rb +0 -1
  16. data/lib/regexp_parser/expression/methods/tests.rb +6 -15
  17. data/lib/regexp_parser/expression/methods/traverse.rb +3 -1
  18. data/lib/regexp_parser/expression/quantifier.rb +2 -2
  19. data/lib/regexp_parser/expression/sequence.rb +3 -6
  20. data/lib/regexp_parser/expression/sequence_operation.rb +2 -6
  21. data/lib/regexp_parser/expression/subexpression.rb +3 -5
  22. data/lib/regexp_parser/lexer.rb +30 -44
  23. data/lib/regexp_parser/parser.rb +47 -24
  24. data/lib/regexp_parser/scanner.rb +1228 -1367
  25. data/lib/regexp_parser/scanner/char_type.rl +0 -3
  26. data/lib/regexp_parser/scanner/properties/long.yml +34 -1
  27. data/lib/regexp_parser/scanner/properties/short.yml +12 -0
  28. data/lib/regexp_parser/scanner/scanner.rl +101 -194
  29. data/lib/regexp_parser/syntax/tokens.rb +2 -10
  30. data/lib/regexp_parser/syntax/tokens/unicode_property.rb +72 -21
  31. data/lib/regexp_parser/syntax/versions/2.6.0.rb +10 -0
  32. data/lib/regexp_parser/syntax/versions/2.6.2.rb +10 -0
  33. data/lib/regexp_parser/syntax/versions/2.6.3.rb +10 -0
  34. data/lib/regexp_parser/version.rb +1 -1
  35. data/regexp_parser.gemspec +3 -3
  36. data/spec/expression/base_spec.rb +94 -0
  37. data/spec/expression/clone_spec.rb +120 -0
  38. data/spec/expression/conditional_spec.rb +89 -0
  39. data/spec/expression/free_space_spec.rb +27 -0
  40. data/spec/expression/methods/match_length_spec.rb +161 -0
  41. data/spec/expression/methods/match_spec.rb +25 -0
  42. data/spec/expression/methods/strfregexp_spec.rb +224 -0
  43. data/spec/expression/methods/tests_spec.rb +99 -0
  44. data/spec/expression/methods/traverse_spec.rb +161 -0
  45. data/spec/expression/options_spec.rb +128 -0
  46. data/spec/expression/root_spec.rb +9 -0
  47. data/spec/expression/sequence_spec.rb +9 -0
  48. data/spec/expression/subexpression_spec.rb +50 -0
  49. data/spec/expression/to_h_spec.rb +26 -0
  50. data/spec/expression/to_s_spec.rb +100 -0
  51. data/spec/lexer/all_spec.rb +22 -0
  52. data/spec/lexer/conditionals_spec.rb +53 -0
  53. data/spec/lexer/delimiters_spec.rb +68 -0
  54. data/spec/lexer/escapes_spec.rb +14 -0
  55. data/spec/lexer/keep_spec.rb +10 -0
  56. data/spec/lexer/literals_spec.rb +89 -0
  57. data/spec/lexer/nesting_spec.rb +99 -0
  58. data/spec/lexer/refcalls_spec.rb +55 -0
  59. data/spec/parser/all_spec.rb +43 -0
  60. data/spec/parser/alternation_spec.rb +88 -0
  61. data/spec/parser/anchors_spec.rb +17 -0
  62. data/spec/parser/conditionals_spec.rb +179 -0
  63. data/spec/parser/errors_spec.rb +30 -0
  64. data/spec/parser/escapes_spec.rb +121 -0
  65. data/spec/parser/free_space_spec.rb +130 -0
  66. data/spec/parser/groups_spec.rb +108 -0
  67. data/spec/parser/keep_spec.rb +6 -0
  68. data/spec/parser/posix_classes_spec.rb +8 -0
  69. data/spec/parser/properties_spec.rb +115 -0
  70. data/spec/parser/quantifiers_spec.rb +52 -0
  71. data/spec/parser/refcalls_spec.rb +112 -0
  72. data/spec/parser/set/intersections_spec.rb +127 -0
  73. data/spec/parser/set/ranges_spec.rb +111 -0
  74. data/spec/parser/sets_spec.rb +178 -0
  75. data/spec/parser/types_spec.rb +18 -0
  76. data/spec/scanner/all_spec.rb +18 -0
  77. data/spec/scanner/anchors_spec.rb +21 -0
  78. data/spec/scanner/conditionals_spec.rb +128 -0
  79. data/spec/scanner/delimiters_spec.rb +52 -0
  80. data/spec/scanner/errors_spec.rb +67 -0
  81. data/spec/scanner/escapes_spec.rb +53 -0
  82. data/spec/scanner/free_space_spec.rb +133 -0
  83. data/spec/scanner/groups_spec.rb +52 -0
  84. data/spec/scanner/keep_spec.rb +10 -0
  85. data/spec/scanner/literals_spec.rb +49 -0
  86. data/spec/scanner/meta_spec.rb +18 -0
  87. data/spec/scanner/properties_spec.rb +64 -0
  88. data/spec/scanner/quantifiers_spec.rb +20 -0
  89. data/spec/scanner/refcalls_spec.rb +36 -0
  90. data/spec/scanner/sets_spec.rb +102 -0
  91. data/spec/scanner/types_spec.rb +14 -0
  92. data/spec/spec_helper.rb +15 -0
  93. data/{test → spec}/support/runner.rb +9 -8
  94. data/spec/support/shared_examples.rb +77 -0
  95. data/{test → spec}/support/warning_extractor.rb +5 -7
  96. data/spec/syntax/syntax_spec.rb +48 -0
  97. data/spec/syntax/syntax_token_map_spec.rb +23 -0
  98. data/spec/syntax/versions/1.8.6_spec.rb +17 -0
  99. data/spec/syntax/versions/1.9.1_spec.rb +10 -0
  100. data/spec/syntax/versions/1.9.3_spec.rb +9 -0
  101. data/spec/syntax/versions/2.0.0_spec.rb +13 -0
  102. data/spec/syntax/versions/2.2.0_spec.rb +9 -0
  103. data/spec/syntax/versions/aliases_spec.rb +37 -0
  104. data/spec/token/token_spec.rb +85 -0
  105. metadata +151 -146
  106. data/test/expression/test_all.rb +0 -12
  107. data/test/expression/test_base.rb +0 -90
  108. data/test/expression/test_clone.rb +0 -89
  109. data/test/expression/test_conditionals.rb +0 -113
  110. data/test/expression/test_free_space.rb +0 -35
  111. data/test/expression/test_set.rb +0 -84
  112. data/test/expression/test_strfregexp.rb +0 -230
  113. data/test/expression/test_subexpression.rb +0 -58
  114. data/test/expression/test_tests.rb +0 -99
  115. data/test/expression/test_to_h.rb +0 -59
  116. data/test/expression/test_to_s.rb +0 -104
  117. data/test/expression/test_traverse.rb +0 -161
  118. data/test/helpers.rb +0 -10
  119. data/test/lexer/test_all.rb +0 -41
  120. data/test/lexer/test_conditionals.rb +0 -127
  121. data/test/lexer/test_keep.rb +0 -24
  122. data/test/lexer/test_literals.rb +0 -130
  123. data/test/lexer/test_nesting.rb +0 -132
  124. data/test/lexer/test_refcalls.rb +0 -56
  125. data/test/parser/set/test_intersections.rb +0 -127
  126. data/test/parser/set/test_ranges.rb +0 -111
  127. data/test/parser/test_all.rb +0 -64
  128. data/test/parser/test_alternation.rb +0 -92
  129. data/test/parser/test_anchors.rb +0 -34
  130. data/test/parser/test_conditionals.rb +0 -187
  131. data/test/parser/test_errors.rb +0 -63
  132. data/test/parser/test_escapes.rb +0 -134
  133. data/test/parser/test_free_space.rb +0 -139
  134. data/test/parser/test_groups.rb +0 -289
  135. data/test/parser/test_keep.rb +0 -21
  136. data/test/parser/test_posix_classes.rb +0 -27
  137. data/test/parser/test_properties.rb +0 -133
  138. data/test/parser/test_quantifiers.rb +0 -301
  139. data/test/parser/test_refcalls.rb +0 -186
  140. data/test/parser/test_sets.rb +0 -179
  141. data/test/parser/test_types.rb +0 -50
  142. data/test/scanner/test_all.rb +0 -38
  143. data/test/scanner/test_anchors.rb +0 -38
  144. data/test/scanner/test_conditionals.rb +0 -184
  145. data/test/scanner/test_errors.rb +0 -91
  146. data/test/scanner/test_escapes.rb +0 -56
  147. data/test/scanner/test_free_space.rb +0 -200
  148. data/test/scanner/test_groups.rb +0 -79
  149. data/test/scanner/test_keep.rb +0 -35
  150. data/test/scanner/test_literals.rb +0 -89
  151. data/test/scanner/test_meta.rb +0 -40
  152. data/test/scanner/test_properties.rb +0 -312
  153. data/test/scanner/test_quantifiers.rb +0 -37
  154. data/test/scanner/test_refcalls.rb +0 -52
  155. data/test/scanner/test_scripts.rb +0 -53
  156. data/test/scanner/test_sets.rb +0 -119
  157. data/test/scanner/test_types.rb +0 -35
  158. data/test/scanner/test_unicode_blocks.rb +0 -30
  159. data/test/support/disable_autotest.rb +0 -8
  160. data/test/syntax/test_all.rb +0 -6
  161. data/test/syntax/test_syntax.rb +0 -61
  162. data/test/syntax/test_syntax_token_map.rb +0 -25
  163. data/test/syntax/versions/test_1.8.rb +0 -55
  164. data/test/syntax/versions/test_1.9.1.rb +0 -36
  165. data/test/syntax/versions/test_1.9.3.rb +0 -32
  166. data/test/syntax/versions/test_2.0.0.rb +0 -37
  167. data/test/syntax/versions/test_2.2.0.rb +0 -32
  168. data/test/syntax/versions/test_aliases.rb +0 -129
  169. data/test/syntax/versions/test_all.rb +0 -5
  170. data/test/test_all.rb +0 -5
  171. data/test/token/test_all.rb +0 -2
  172. data/test/token/test_token.rb +0 -107
@@ -1,36 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_1 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.1'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- escape: [
13
- Escape::Hex + Escape::Octal + Escape::Unicode
14
- ].flatten,
15
-
16
- type: [
17
- CharacterType::Hex
18
- ].flatten,
19
-
20
- quantifier: [
21
- Quantifier::Greedy + Quantifier::Reluctant + Quantifier::Possessive
22
- ].flatten,
23
- },
24
- }
25
-
26
- tests.each do |method, types|
27
- types.each do |type, tokens|
28
- tokens.each do |token|
29
- define_method "test_syntax_V1_9_1_#{method}_#{type}_#{token}" do
30
- assert_equal true, @syntax.implements?(type, token)
31
- end
32
- end
33
- end
34
- end
35
-
36
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_3 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.3'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V1_9_3_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,37 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_0_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.0.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Age_V2_0_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Age_V2_0_0
18
- ].flatten,
19
- },
20
- excludes: {
21
- property: [:newline],
22
- nonproperty: [:newline],
23
- }
24
- }
25
-
26
- tests.each do |method, types|
27
- expected = method == :excludes ? false : true
28
-
29
- types.each do |type, tokens|
30
- tokens.each do |token|
31
- define_method "test_syntax_ruby_V2_0_0_#{method}_#{type}_#{token}" do
32
- assert_equal expected, @syntax.implements?(type, token)
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_2_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.2.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V2_2_0_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,129 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxAliases < Test::Unit::TestCase
4
- # 1.8 syntax
5
- def test_syntax_alias_1_8_6
6
- syntax = Regexp::Syntax.new 'ruby/1.8.6'
7
-
8
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
9
- end
10
-
11
- def test_syntax_alias_1_8_alias
12
- syntax = Regexp::Syntax.new 'ruby/1.8'
13
-
14
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
15
- end
16
-
17
- # 1.9 syntax
18
- def test_syntax_alias_1_9_1
19
- syntax = Regexp::Syntax.new 'ruby/1.9.1'
20
-
21
- assert syntax.is_a?(Regexp::Syntax::V1_9_1)
22
- end
23
-
24
- def test_syntax_alias_1_9_alias
25
- syntax = Regexp::Syntax.new 'ruby/1.9'
26
-
27
- assert syntax.is_a?(Regexp::Syntax::V1_9_3)
28
- end
29
-
30
- # 2.0 syntax
31
- def test_syntax_alias_2_0_0
32
- syntax = Regexp::Syntax.new 'ruby/2.0.0'
33
-
34
- assert syntax.is_a?(Regexp::Syntax::V1_9)
35
- end
36
-
37
- def test_syntax_alias_2_0_alias
38
- syntax = Regexp::Syntax.new 'ruby/2.0'
39
-
40
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
41
- end
42
-
43
- # 2.1 syntax
44
- def test_syntax_alias_2_1_alias
45
- syntax = Regexp::Syntax.new 'ruby/2.1'
46
-
47
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
48
- end
49
-
50
- # 2.2 syntax
51
- def test_syntax_alias_2_2_0
52
- syntax = Regexp::Syntax.new 'ruby/2.2.0'
53
-
54
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
55
- end
56
-
57
- def test_syntax_alias_2_2_10
58
- syntax = Regexp::Syntax.new 'ruby/2.2.10'
59
-
60
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
61
- end
62
-
63
- def test_syntax_alias_2_2_alias
64
- syntax = Regexp::Syntax.new 'ruby/2.2'
65
-
66
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
67
- end
68
-
69
- # 2.3 syntax
70
- def test_syntax_alias_2_3_0
71
- syntax = Regexp::Syntax.new 'ruby/2.3.0'
72
-
73
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
74
- end
75
-
76
- def test_syntax_alias_2_3
77
- syntax = Regexp::Syntax.new 'ruby/2.3'
78
-
79
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
80
- end
81
-
82
- # 2.4 syntax
83
- def test_syntax_alias_2_4_0
84
- syntax = Regexp::Syntax.new 'ruby/2.4.0'
85
-
86
- assert syntax.is_a?(Regexp::Syntax::V2_4_0)
87
- end
88
-
89
- def test_syntax_alias_2_4_1
90
- syntax = Regexp::Syntax.new 'ruby/2.4.1'
91
-
92
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
93
- end
94
-
95
- # 2.5 syntax
96
- def test_syntax_alias_2_5_0
97
- syntax = Regexp::Syntax.new 'ruby/2.5.0'
98
-
99
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
100
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
101
- end
102
-
103
- def test_syntax_alias_2_5
104
- syntax = Regexp::Syntax.new 'ruby/2.5'
105
-
106
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
107
- end
108
-
109
- # 2.6 syntax
110
- def test_syntax_alias_2_6_0
111
- syntax = Regexp::Syntax.new 'ruby/2.6.0'
112
-
113
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
114
- assert syntax.is_a?(Regexp::Syntax::V2_6_0)
115
- end
116
-
117
- def test_syntax_alias_2_6
118
- syntax = Regexp::Syntax.new 'ruby/2.6'
119
-
120
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
121
- end
122
-
123
- def test_future_alias_warning
124
- _, stderr_output = capture_output { Regexp::Syntax.new 'ruby/5.0' }
125
-
126
- assert_match(/This library .* but you are running .* \(feature set of .*\)/,
127
- stderr_output)
128
- end
129
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- %w{aliases 1.8 1.9.1 1.9.3 2.0.0 2.2.0}.each do|tc|
4
- require File.expand_path("../test_#{tc}", __FILE__)
5
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../helpers", __FILE__)
2
-
3
- %w{ scanner syntax token lexer parser expression }.each do |subject|
4
- require File.expand_path("../#{subject}/test_all", __FILE__)
5
- end
@@ -1,2 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
- require File.expand_path("../test_token", __FILE__)
@@ -1,107 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
-
3
- class Token < Test::Unit::TestCase
4
-
5
- def test_token_offset
6
- regexp = /ab?cd/
7
- tokens = RL.lex(regexp)
8
-
9
- assert_equal 'b', tokens[1].text
10
- assert_equal [1, 2], tokens[1].offset
11
-
12
- assert_equal '?', tokens[2].text
13
- assert_equal [2, 3], tokens[2].offset
14
-
15
- assert_equal 'cd', tokens[3].text
16
- assert_equal [3, 5], tokens[3].offset
17
- end
18
-
19
- def test_token_length
20
- regexp = /abc?def/
21
- tokens = RL.lex(regexp)
22
-
23
- assert_equal 'ab', tokens[0].text
24
- assert_equal 2, tokens[0].length
25
-
26
- assert_equal 'c', tokens[1].text
27
- assert_equal 1, tokens[1].length
28
-
29
- assert_equal '?', tokens[2].text
30
- assert_equal 1, tokens[2].length
31
-
32
- assert_equal 'def', tokens[3].text
33
- assert_equal 3, tokens[3].length
34
- end
35
-
36
- def test_token_to_h
37
- regexp = /abc?def/
38
- tokens = RL.lex(regexp)
39
-
40
- assert_equal 'ab', tokens[0].text
41
- assert_equal({
42
- :type => :literal,
43
- :token => :literal,
44
- :text => 'ab',
45
- :ts => 0,
46
- :te => 2,
47
- :level => 0,
48
- :set_level => 0,
49
- :conditional_level => 0
50
- }, tokens[0].to_h
51
- )
52
-
53
- assert_equal '?', tokens[2].text
54
- assert_equal({
55
- :type => :quantifier,
56
- :token => :zero_or_one,
57
- :text => '?',
58
- :ts => 3,
59
- :te => 4,
60
- :level => 0,
61
- :set_level => 0,
62
- :conditional_level => 0
63
- }, tokens[2].to_h
64
- )
65
- end
66
-
67
- def test_token_next
68
- regexp = /a+b?c*d{2,3}/
69
- tokens = RL.lex(regexp)
70
-
71
- a = tokens.first
72
- assert_equal 'a', a.text
73
-
74
- plus = a.next
75
- assert_equal '+', plus.text
76
-
77
- b = plus.next
78
- assert_equal 'b', b.text
79
-
80
- interval = tokens.last
81
- assert_equal '{2,3}', interval.text
82
-
83
- assert_equal nil, interval.next
84
- end
85
-
86
- def test_token_previous
87
- regexp = /a+b?c*d{2,3}/
88
- tokens = RL.lex(regexp)
89
-
90
- interval = tokens.last
91
- assert_equal '{2,3}', interval.text
92
-
93
- d = interval.previous
94
- assert_equal 'd', d.text
95
-
96
- star = d.previous
97
- assert_equal '*', star.text
98
-
99
- c = star.previous
100
- assert_equal 'c', c.text
101
-
102
- a = tokens.first
103
- assert_equal 'a', a.text
104
- assert_equal nil, a.previous
105
- end
106
-
107
- end