regexp_parser 1.4.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +66 -1
  3. data/Gemfile +3 -3
  4. data/README.md +11 -18
  5. data/Rakefile +3 -4
  6. data/lib/regexp_parser/expression.rb +28 -53
  7. data/lib/regexp_parser/expression/classes/backref.rb +18 -10
  8. data/lib/regexp_parser/expression/classes/conditional.rb +7 -2
  9. data/lib/regexp_parser/expression/classes/escape.rb +0 -4
  10. data/lib/regexp_parser/expression/classes/group.rb +4 -2
  11. data/lib/regexp_parser/expression/classes/keep.rb +1 -3
  12. data/lib/regexp_parser/expression/methods/match.rb +13 -0
  13. data/lib/regexp_parser/expression/methods/match_length.rb +172 -0
  14. data/lib/regexp_parser/expression/methods/options.rb +35 -0
  15. data/lib/regexp_parser/expression/methods/strfregexp.rb +0 -1
  16. data/lib/regexp_parser/expression/methods/tests.rb +6 -15
  17. data/lib/regexp_parser/expression/methods/traverse.rb +3 -1
  18. data/lib/regexp_parser/expression/quantifier.rb +2 -2
  19. data/lib/regexp_parser/expression/sequence.rb +3 -6
  20. data/lib/regexp_parser/expression/sequence_operation.rb +2 -6
  21. data/lib/regexp_parser/expression/subexpression.rb +3 -5
  22. data/lib/regexp_parser/lexer.rb +30 -44
  23. data/lib/regexp_parser/parser.rb +47 -24
  24. data/lib/regexp_parser/scanner.rb +1228 -1367
  25. data/lib/regexp_parser/scanner/char_type.rl +0 -3
  26. data/lib/regexp_parser/scanner/properties/long.yml +15 -1
  27. data/lib/regexp_parser/scanner/properties/short.yml +5 -0
  28. data/lib/regexp_parser/scanner/scanner.rl +101 -194
  29. data/lib/regexp_parser/syntax/tokens.rb +2 -10
  30. data/lib/regexp_parser/syntax/tokens/unicode_property.rb +30 -0
  31. data/lib/regexp_parser/syntax/versions/2.6.2.rb +10 -0
  32. data/lib/regexp_parser/syntax/versions/2.6.3.rb +10 -0
  33. data/lib/regexp_parser/version.rb +1 -1
  34. data/regexp_parser.gemspec +2 -2
  35. data/spec/expression/base_spec.rb +94 -0
  36. data/spec/expression/clone_spec.rb +120 -0
  37. data/spec/expression/conditional_spec.rb +89 -0
  38. data/spec/expression/free_space_spec.rb +27 -0
  39. data/spec/expression/methods/match_length_spec.rb +161 -0
  40. data/spec/expression/methods/match_spec.rb +25 -0
  41. data/spec/expression/methods/strfregexp_spec.rb +224 -0
  42. data/spec/expression/methods/tests_spec.rb +99 -0
  43. data/spec/expression/methods/traverse_spec.rb +161 -0
  44. data/spec/expression/options_spec.rb +128 -0
  45. data/spec/expression/root_spec.rb +9 -0
  46. data/spec/expression/sequence_spec.rb +9 -0
  47. data/spec/expression/subexpression_spec.rb +50 -0
  48. data/spec/expression/to_h_spec.rb +26 -0
  49. data/spec/expression/to_s_spec.rb +100 -0
  50. data/spec/lexer/all_spec.rb +22 -0
  51. data/spec/lexer/conditionals_spec.rb +53 -0
  52. data/spec/lexer/delimiters_spec.rb +68 -0
  53. data/spec/lexer/escapes_spec.rb +14 -0
  54. data/spec/lexer/keep_spec.rb +10 -0
  55. data/spec/lexer/literals_spec.rb +89 -0
  56. data/spec/lexer/nesting_spec.rb +99 -0
  57. data/spec/lexer/refcalls_spec.rb +55 -0
  58. data/spec/parser/all_spec.rb +43 -0
  59. data/spec/parser/alternation_spec.rb +88 -0
  60. data/spec/parser/anchors_spec.rb +17 -0
  61. data/spec/parser/conditionals_spec.rb +179 -0
  62. data/spec/parser/errors_spec.rb +30 -0
  63. data/spec/parser/escapes_spec.rb +121 -0
  64. data/spec/parser/free_space_spec.rb +130 -0
  65. data/spec/parser/groups_spec.rb +108 -0
  66. data/spec/parser/keep_spec.rb +6 -0
  67. data/spec/parser/posix_classes_spec.rb +8 -0
  68. data/spec/parser/properties_spec.rb +115 -0
  69. data/spec/parser/quantifiers_spec.rb +52 -0
  70. data/spec/parser/refcalls_spec.rb +112 -0
  71. data/spec/parser/set/intersections_spec.rb +127 -0
  72. data/spec/parser/set/ranges_spec.rb +111 -0
  73. data/spec/parser/sets_spec.rb +178 -0
  74. data/spec/parser/types_spec.rb +18 -0
  75. data/spec/scanner/all_spec.rb +18 -0
  76. data/spec/scanner/anchors_spec.rb +21 -0
  77. data/spec/scanner/conditionals_spec.rb +128 -0
  78. data/spec/scanner/delimiters_spec.rb +52 -0
  79. data/spec/scanner/errors_spec.rb +67 -0
  80. data/spec/scanner/escapes_spec.rb +53 -0
  81. data/spec/scanner/free_space_spec.rb +133 -0
  82. data/spec/scanner/groups_spec.rb +52 -0
  83. data/spec/scanner/keep_spec.rb +10 -0
  84. data/spec/scanner/literals_spec.rb +49 -0
  85. data/spec/scanner/meta_spec.rb +18 -0
  86. data/spec/scanner/properties_spec.rb +64 -0
  87. data/spec/scanner/quantifiers_spec.rb +20 -0
  88. data/spec/scanner/refcalls_spec.rb +36 -0
  89. data/spec/scanner/sets_spec.rb +102 -0
  90. data/spec/scanner/types_spec.rb +14 -0
  91. data/spec/spec_helper.rb +15 -0
  92. data/{test → spec}/support/runner.rb +9 -8
  93. data/spec/support/shared_examples.rb +77 -0
  94. data/{test → spec}/support/warning_extractor.rb +5 -7
  95. data/spec/syntax/syntax_spec.rb +48 -0
  96. data/spec/syntax/syntax_token_map_spec.rb +23 -0
  97. data/spec/syntax/versions/1.8.6_spec.rb +17 -0
  98. data/spec/syntax/versions/1.9.1_spec.rb +10 -0
  99. data/spec/syntax/versions/1.9.3_spec.rb +9 -0
  100. data/spec/syntax/versions/2.0.0_spec.rb +13 -0
  101. data/spec/syntax/versions/2.2.0_spec.rb +9 -0
  102. data/spec/syntax/versions/aliases_spec.rb +37 -0
  103. data/spec/token/token_spec.rb +85 -0
  104. metadata +149 -144
  105. data/test/expression/test_all.rb +0 -12
  106. data/test/expression/test_base.rb +0 -90
  107. data/test/expression/test_clone.rb +0 -89
  108. data/test/expression/test_conditionals.rb +0 -113
  109. data/test/expression/test_free_space.rb +0 -35
  110. data/test/expression/test_set.rb +0 -84
  111. data/test/expression/test_strfregexp.rb +0 -230
  112. data/test/expression/test_subexpression.rb +0 -58
  113. data/test/expression/test_tests.rb +0 -99
  114. data/test/expression/test_to_h.rb +0 -59
  115. data/test/expression/test_to_s.rb +0 -104
  116. data/test/expression/test_traverse.rb +0 -161
  117. data/test/helpers.rb +0 -10
  118. data/test/lexer/test_all.rb +0 -41
  119. data/test/lexer/test_conditionals.rb +0 -127
  120. data/test/lexer/test_keep.rb +0 -24
  121. data/test/lexer/test_literals.rb +0 -130
  122. data/test/lexer/test_nesting.rb +0 -132
  123. data/test/lexer/test_refcalls.rb +0 -56
  124. data/test/parser/set/test_intersections.rb +0 -127
  125. data/test/parser/set/test_ranges.rb +0 -111
  126. data/test/parser/test_all.rb +0 -64
  127. data/test/parser/test_alternation.rb +0 -92
  128. data/test/parser/test_anchors.rb +0 -34
  129. data/test/parser/test_conditionals.rb +0 -187
  130. data/test/parser/test_errors.rb +0 -63
  131. data/test/parser/test_escapes.rb +0 -134
  132. data/test/parser/test_free_space.rb +0 -139
  133. data/test/parser/test_groups.rb +0 -289
  134. data/test/parser/test_keep.rb +0 -21
  135. data/test/parser/test_posix_classes.rb +0 -27
  136. data/test/parser/test_properties.rb +0 -134
  137. data/test/parser/test_quantifiers.rb +0 -301
  138. data/test/parser/test_refcalls.rb +0 -186
  139. data/test/parser/test_sets.rb +0 -179
  140. data/test/parser/test_types.rb +0 -50
  141. data/test/scanner/test_all.rb +0 -38
  142. data/test/scanner/test_anchors.rb +0 -38
  143. data/test/scanner/test_conditionals.rb +0 -184
  144. data/test/scanner/test_errors.rb +0 -91
  145. data/test/scanner/test_escapes.rb +0 -56
  146. data/test/scanner/test_free_space.rb +0 -200
  147. data/test/scanner/test_groups.rb +0 -79
  148. data/test/scanner/test_keep.rb +0 -35
  149. data/test/scanner/test_literals.rb +0 -89
  150. data/test/scanner/test_meta.rb +0 -40
  151. data/test/scanner/test_properties.rb +0 -312
  152. data/test/scanner/test_quantifiers.rb +0 -37
  153. data/test/scanner/test_refcalls.rb +0 -52
  154. data/test/scanner/test_scripts.rb +0 -53
  155. data/test/scanner/test_sets.rb +0 -119
  156. data/test/scanner/test_types.rb +0 -35
  157. data/test/scanner/test_unicode_blocks.rb +0 -30
  158. data/test/support/disable_autotest.rb +0 -8
  159. data/test/syntax/test_all.rb +0 -6
  160. data/test/syntax/test_syntax.rb +0 -61
  161. data/test/syntax/test_syntax_token_map.rb +0 -25
  162. data/test/syntax/versions/test_1.8.rb +0 -55
  163. data/test/syntax/versions/test_1.9.1.rb +0 -36
  164. data/test/syntax/versions/test_1.9.3.rb +0 -32
  165. data/test/syntax/versions/test_2.0.0.rb +0 -37
  166. data/test/syntax/versions/test_2.2.0.rb +0 -32
  167. data/test/syntax/versions/test_aliases.rb +0 -129
  168. data/test/syntax/versions/test_all.rb +0 -5
  169. data/test/test_all.rb +0 -5
  170. data/test/token/test_all.rb +0 -2
  171. data/test/token/test_token.rb +0 -107
@@ -1,36 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_1 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.1'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- escape: [
13
- Escape::Hex + Escape::Octal + Escape::Unicode
14
- ].flatten,
15
-
16
- type: [
17
- CharacterType::Hex
18
- ].flatten,
19
-
20
- quantifier: [
21
- Quantifier::Greedy + Quantifier::Reluctant + Quantifier::Possessive
22
- ].flatten,
23
- },
24
- }
25
-
26
- tests.each do |method, types|
27
- types.each do |type, tokens|
28
- tokens.each do |token|
29
- define_method "test_syntax_V1_9_1_#{method}_#{type}_#{token}" do
30
- assert_equal true, @syntax.implements?(type, token)
31
- end
32
- end
33
- end
34
- end
35
-
36
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_3 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.3'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V1_9_3_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,37 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_0_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.0.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Age_V2_0_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Age_V2_0_0
18
- ].flatten,
19
- },
20
- excludes: {
21
- property: [:newline],
22
- nonproperty: [:newline],
23
- }
24
- }
25
-
26
- tests.each do |method, types|
27
- expected = method == :excludes ? false : true
28
-
29
- types.each do |type, tokens|
30
- tokens.each do |token|
31
- define_method "test_syntax_ruby_V2_0_0_#{method}_#{type}_#{token}" do
32
- assert_equal expected, @syntax.implements?(type, token)
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_2_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.2.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V2_2_0_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,129 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxAliases < Test::Unit::TestCase
4
- # 1.8 syntax
5
- def test_syntax_alias_1_8_6
6
- syntax = Regexp::Syntax.new 'ruby/1.8.6'
7
-
8
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
9
- end
10
-
11
- def test_syntax_alias_1_8_alias
12
- syntax = Regexp::Syntax.new 'ruby/1.8'
13
-
14
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
15
- end
16
-
17
- # 1.9 syntax
18
- def test_syntax_alias_1_9_1
19
- syntax = Regexp::Syntax.new 'ruby/1.9.1'
20
-
21
- assert syntax.is_a?(Regexp::Syntax::V1_9_1)
22
- end
23
-
24
- def test_syntax_alias_1_9_alias
25
- syntax = Regexp::Syntax.new 'ruby/1.9'
26
-
27
- assert syntax.is_a?(Regexp::Syntax::V1_9_3)
28
- end
29
-
30
- # 2.0 syntax
31
- def test_syntax_alias_2_0_0
32
- syntax = Regexp::Syntax.new 'ruby/2.0.0'
33
-
34
- assert syntax.is_a?(Regexp::Syntax::V1_9)
35
- end
36
-
37
- def test_syntax_alias_2_0_alias
38
- syntax = Regexp::Syntax.new 'ruby/2.0'
39
-
40
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
41
- end
42
-
43
- # 2.1 syntax
44
- def test_syntax_alias_2_1_alias
45
- syntax = Regexp::Syntax.new 'ruby/2.1'
46
-
47
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
48
- end
49
-
50
- # 2.2 syntax
51
- def test_syntax_alias_2_2_0
52
- syntax = Regexp::Syntax.new 'ruby/2.2.0'
53
-
54
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
55
- end
56
-
57
- def test_syntax_alias_2_2_10
58
- syntax = Regexp::Syntax.new 'ruby/2.2.10'
59
-
60
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
61
- end
62
-
63
- def test_syntax_alias_2_2_alias
64
- syntax = Regexp::Syntax.new 'ruby/2.2'
65
-
66
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
67
- end
68
-
69
- # 2.3 syntax
70
- def test_syntax_alias_2_3_0
71
- syntax = Regexp::Syntax.new 'ruby/2.3.0'
72
-
73
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
74
- end
75
-
76
- def test_syntax_alias_2_3
77
- syntax = Regexp::Syntax.new 'ruby/2.3'
78
-
79
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
80
- end
81
-
82
- # 2.4 syntax
83
- def test_syntax_alias_2_4_0
84
- syntax = Regexp::Syntax.new 'ruby/2.4.0'
85
-
86
- assert syntax.is_a?(Regexp::Syntax::V2_4_0)
87
- end
88
-
89
- def test_syntax_alias_2_4_1
90
- syntax = Regexp::Syntax.new 'ruby/2.4.1'
91
-
92
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
93
- end
94
-
95
- # 2.5 syntax
96
- def test_syntax_alias_2_5_0
97
- syntax = Regexp::Syntax.new 'ruby/2.5.0'
98
-
99
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
100
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
101
- end
102
-
103
- def test_syntax_alias_2_5
104
- syntax = Regexp::Syntax.new 'ruby/2.5'
105
-
106
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
107
- end
108
-
109
- # 2.6 syntax
110
- def test_syntax_alias_2_6_0
111
- syntax = Regexp::Syntax.new 'ruby/2.6.0'
112
-
113
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
114
- assert syntax.is_a?(Regexp::Syntax::V2_6_0)
115
- end
116
-
117
- def test_syntax_alias_2_6
118
- syntax = Regexp::Syntax.new 'ruby/2.6'
119
-
120
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
121
- end
122
-
123
- def test_future_alias_warning
124
- _, stderr_output = capture_output { Regexp::Syntax.new 'ruby/5.0' }
125
-
126
- assert_match(/This library .* but you are running .* \(feature set of .*\)/,
127
- stderr_output)
128
- end
129
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- %w{aliases 1.8 1.9.1 1.9.3 2.0.0 2.2.0}.each do|tc|
4
- require File.expand_path("../test_#{tc}", __FILE__)
5
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../helpers", __FILE__)
2
-
3
- %w{ scanner syntax token lexer parser expression }.each do |subject|
4
- require File.expand_path("../#{subject}/test_all", __FILE__)
5
- end
@@ -1,2 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
- require File.expand_path("../test_token", __FILE__)
@@ -1,107 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
-
3
- class Token < Test::Unit::TestCase
4
-
5
- def test_token_offset
6
- regexp = /ab?cd/
7
- tokens = RL.lex(regexp)
8
-
9
- assert_equal 'b', tokens[1].text
10
- assert_equal [1, 2], tokens[1].offset
11
-
12
- assert_equal '?', tokens[2].text
13
- assert_equal [2, 3], tokens[2].offset
14
-
15
- assert_equal 'cd', tokens[3].text
16
- assert_equal [3, 5], tokens[3].offset
17
- end
18
-
19
- def test_token_length
20
- regexp = /abc?def/
21
- tokens = RL.lex(regexp)
22
-
23
- assert_equal 'ab', tokens[0].text
24
- assert_equal 2, tokens[0].length
25
-
26
- assert_equal 'c', tokens[1].text
27
- assert_equal 1, tokens[1].length
28
-
29
- assert_equal '?', tokens[2].text
30
- assert_equal 1, tokens[2].length
31
-
32
- assert_equal 'def', tokens[3].text
33
- assert_equal 3, tokens[3].length
34
- end
35
-
36
- def test_token_to_h
37
- regexp = /abc?def/
38
- tokens = RL.lex(regexp)
39
-
40
- assert_equal 'ab', tokens[0].text
41
- assert_equal({
42
- :type => :literal,
43
- :token => :literal,
44
- :text => 'ab',
45
- :ts => 0,
46
- :te => 2,
47
- :level => 0,
48
- :set_level => 0,
49
- :conditional_level => 0
50
- }, tokens[0].to_h
51
- )
52
-
53
- assert_equal '?', tokens[2].text
54
- assert_equal({
55
- :type => :quantifier,
56
- :token => :zero_or_one,
57
- :text => '?',
58
- :ts => 3,
59
- :te => 4,
60
- :level => 0,
61
- :set_level => 0,
62
- :conditional_level => 0
63
- }, tokens[2].to_h
64
- )
65
- end
66
-
67
- def test_token_next
68
- regexp = /a+b?c*d{2,3}/
69
- tokens = RL.lex(regexp)
70
-
71
- a = tokens.first
72
- assert_equal 'a', a.text
73
-
74
- plus = a.next
75
- assert_equal '+', plus.text
76
-
77
- b = plus.next
78
- assert_equal 'b', b.text
79
-
80
- interval = tokens.last
81
- assert_equal '{2,3}', interval.text
82
-
83
- assert_equal nil, interval.next
84
- end
85
-
86
- def test_token_previous
87
- regexp = /a+b?c*d{2,3}/
88
- tokens = RL.lex(regexp)
89
-
90
- interval = tokens.last
91
- assert_equal '{2,3}', interval.text
92
-
93
- d = interval.previous
94
- assert_equal 'd', d.text
95
-
96
- star = d.previous
97
- assert_equal '*', star.text
98
-
99
- c = star.previous
100
- assert_equal 'c', c.text
101
-
102
- a = tokens.first
103
- assert_equal 'a', a.text
104
- assert_equal nil, a.previous
105
- end
106
-
107
- end