regexp_parser 1.3.0 → 1.7.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (172) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -1
  3. data/Gemfile +3 -3
  4. data/README.md +12 -19
  5. data/Rakefile +3 -4
  6. data/lib/regexp_parser/expression.rb +28 -53
  7. data/lib/regexp_parser/expression/classes/backref.rb +18 -10
  8. data/lib/regexp_parser/expression/classes/conditional.rb +7 -2
  9. data/lib/regexp_parser/expression/classes/escape.rb +0 -4
  10. data/lib/regexp_parser/expression/classes/group.rb +4 -2
  11. data/lib/regexp_parser/expression/classes/keep.rb +1 -3
  12. data/lib/regexp_parser/expression/methods/match.rb +13 -0
  13. data/lib/regexp_parser/expression/methods/match_length.rb +172 -0
  14. data/lib/regexp_parser/expression/methods/options.rb +35 -0
  15. data/lib/regexp_parser/expression/methods/strfregexp.rb +0 -1
  16. data/lib/regexp_parser/expression/methods/tests.rb +6 -15
  17. data/lib/regexp_parser/expression/methods/traverse.rb +3 -1
  18. data/lib/regexp_parser/expression/quantifier.rb +2 -2
  19. data/lib/regexp_parser/expression/sequence.rb +3 -6
  20. data/lib/regexp_parser/expression/sequence_operation.rb +2 -6
  21. data/lib/regexp_parser/expression/subexpression.rb +3 -5
  22. data/lib/regexp_parser/lexer.rb +30 -44
  23. data/lib/regexp_parser/parser.rb +47 -24
  24. data/lib/regexp_parser/scanner.rb +1228 -1367
  25. data/lib/regexp_parser/scanner/char_type.rl +0 -3
  26. data/lib/regexp_parser/scanner/properties/long.yml +34 -1
  27. data/lib/regexp_parser/scanner/properties/short.yml +12 -0
  28. data/lib/regexp_parser/scanner/scanner.rl +101 -194
  29. data/lib/regexp_parser/syntax/tokens.rb +2 -10
  30. data/lib/regexp_parser/syntax/tokens/unicode_property.rb +72 -21
  31. data/lib/regexp_parser/syntax/versions/2.6.0.rb +10 -0
  32. data/lib/regexp_parser/syntax/versions/2.6.2.rb +10 -0
  33. data/lib/regexp_parser/syntax/versions/2.6.3.rb +10 -0
  34. data/lib/regexp_parser/version.rb +1 -1
  35. data/regexp_parser.gemspec +3 -3
  36. data/spec/expression/base_spec.rb +94 -0
  37. data/spec/expression/clone_spec.rb +120 -0
  38. data/spec/expression/conditional_spec.rb +89 -0
  39. data/spec/expression/free_space_spec.rb +27 -0
  40. data/spec/expression/methods/match_length_spec.rb +161 -0
  41. data/spec/expression/methods/match_spec.rb +25 -0
  42. data/spec/expression/methods/strfregexp_spec.rb +224 -0
  43. data/spec/expression/methods/tests_spec.rb +99 -0
  44. data/spec/expression/methods/traverse_spec.rb +161 -0
  45. data/spec/expression/options_spec.rb +128 -0
  46. data/spec/expression/root_spec.rb +9 -0
  47. data/spec/expression/sequence_spec.rb +9 -0
  48. data/spec/expression/subexpression_spec.rb +50 -0
  49. data/spec/expression/to_h_spec.rb +26 -0
  50. data/spec/expression/to_s_spec.rb +100 -0
  51. data/spec/lexer/all_spec.rb +22 -0
  52. data/spec/lexer/conditionals_spec.rb +53 -0
  53. data/spec/lexer/delimiters_spec.rb +68 -0
  54. data/spec/lexer/escapes_spec.rb +14 -0
  55. data/spec/lexer/keep_spec.rb +10 -0
  56. data/spec/lexer/literals_spec.rb +89 -0
  57. data/spec/lexer/nesting_spec.rb +99 -0
  58. data/spec/lexer/refcalls_spec.rb +55 -0
  59. data/spec/parser/all_spec.rb +43 -0
  60. data/spec/parser/alternation_spec.rb +88 -0
  61. data/spec/parser/anchors_spec.rb +17 -0
  62. data/spec/parser/conditionals_spec.rb +179 -0
  63. data/spec/parser/errors_spec.rb +30 -0
  64. data/spec/parser/escapes_spec.rb +121 -0
  65. data/spec/parser/free_space_spec.rb +130 -0
  66. data/spec/parser/groups_spec.rb +108 -0
  67. data/spec/parser/keep_spec.rb +6 -0
  68. data/spec/parser/posix_classes_spec.rb +8 -0
  69. data/spec/parser/properties_spec.rb +115 -0
  70. data/spec/parser/quantifiers_spec.rb +52 -0
  71. data/spec/parser/refcalls_spec.rb +112 -0
  72. data/spec/parser/set/intersections_spec.rb +127 -0
  73. data/spec/parser/set/ranges_spec.rb +111 -0
  74. data/spec/parser/sets_spec.rb +178 -0
  75. data/spec/parser/types_spec.rb +18 -0
  76. data/spec/scanner/all_spec.rb +18 -0
  77. data/spec/scanner/anchors_spec.rb +21 -0
  78. data/spec/scanner/conditionals_spec.rb +128 -0
  79. data/spec/scanner/delimiters_spec.rb +52 -0
  80. data/spec/scanner/errors_spec.rb +67 -0
  81. data/spec/scanner/escapes_spec.rb +53 -0
  82. data/spec/scanner/free_space_spec.rb +133 -0
  83. data/spec/scanner/groups_spec.rb +52 -0
  84. data/spec/scanner/keep_spec.rb +10 -0
  85. data/spec/scanner/literals_spec.rb +49 -0
  86. data/spec/scanner/meta_spec.rb +18 -0
  87. data/spec/scanner/properties_spec.rb +64 -0
  88. data/spec/scanner/quantifiers_spec.rb +20 -0
  89. data/spec/scanner/refcalls_spec.rb +36 -0
  90. data/spec/scanner/sets_spec.rb +102 -0
  91. data/spec/scanner/types_spec.rb +14 -0
  92. data/spec/spec_helper.rb +15 -0
  93. data/{test → spec}/support/runner.rb +9 -8
  94. data/spec/support/shared_examples.rb +77 -0
  95. data/{test → spec}/support/warning_extractor.rb +5 -7
  96. data/spec/syntax/syntax_spec.rb +48 -0
  97. data/spec/syntax/syntax_token_map_spec.rb +23 -0
  98. data/spec/syntax/versions/1.8.6_spec.rb +17 -0
  99. data/spec/syntax/versions/1.9.1_spec.rb +10 -0
  100. data/spec/syntax/versions/1.9.3_spec.rb +9 -0
  101. data/spec/syntax/versions/2.0.0_spec.rb +13 -0
  102. data/spec/syntax/versions/2.2.0_spec.rb +9 -0
  103. data/spec/syntax/versions/aliases_spec.rb +37 -0
  104. data/spec/token/token_spec.rb +85 -0
  105. metadata +151 -146
  106. data/test/expression/test_all.rb +0 -12
  107. data/test/expression/test_base.rb +0 -90
  108. data/test/expression/test_clone.rb +0 -89
  109. data/test/expression/test_conditionals.rb +0 -113
  110. data/test/expression/test_free_space.rb +0 -35
  111. data/test/expression/test_set.rb +0 -84
  112. data/test/expression/test_strfregexp.rb +0 -230
  113. data/test/expression/test_subexpression.rb +0 -58
  114. data/test/expression/test_tests.rb +0 -99
  115. data/test/expression/test_to_h.rb +0 -59
  116. data/test/expression/test_to_s.rb +0 -104
  117. data/test/expression/test_traverse.rb +0 -161
  118. data/test/helpers.rb +0 -10
  119. data/test/lexer/test_all.rb +0 -41
  120. data/test/lexer/test_conditionals.rb +0 -127
  121. data/test/lexer/test_keep.rb +0 -24
  122. data/test/lexer/test_literals.rb +0 -130
  123. data/test/lexer/test_nesting.rb +0 -132
  124. data/test/lexer/test_refcalls.rb +0 -56
  125. data/test/parser/set/test_intersections.rb +0 -127
  126. data/test/parser/set/test_ranges.rb +0 -111
  127. data/test/parser/test_all.rb +0 -64
  128. data/test/parser/test_alternation.rb +0 -92
  129. data/test/parser/test_anchors.rb +0 -34
  130. data/test/parser/test_conditionals.rb +0 -187
  131. data/test/parser/test_errors.rb +0 -63
  132. data/test/parser/test_escapes.rb +0 -134
  133. data/test/parser/test_free_space.rb +0 -139
  134. data/test/parser/test_groups.rb +0 -289
  135. data/test/parser/test_keep.rb +0 -21
  136. data/test/parser/test_posix_classes.rb +0 -27
  137. data/test/parser/test_properties.rb +0 -133
  138. data/test/parser/test_quantifiers.rb +0 -301
  139. data/test/parser/test_refcalls.rb +0 -186
  140. data/test/parser/test_sets.rb +0 -179
  141. data/test/parser/test_types.rb +0 -50
  142. data/test/scanner/test_all.rb +0 -38
  143. data/test/scanner/test_anchors.rb +0 -38
  144. data/test/scanner/test_conditionals.rb +0 -184
  145. data/test/scanner/test_errors.rb +0 -91
  146. data/test/scanner/test_escapes.rb +0 -56
  147. data/test/scanner/test_free_space.rb +0 -200
  148. data/test/scanner/test_groups.rb +0 -79
  149. data/test/scanner/test_keep.rb +0 -35
  150. data/test/scanner/test_literals.rb +0 -89
  151. data/test/scanner/test_meta.rb +0 -40
  152. data/test/scanner/test_properties.rb +0 -312
  153. data/test/scanner/test_quantifiers.rb +0 -37
  154. data/test/scanner/test_refcalls.rb +0 -52
  155. data/test/scanner/test_scripts.rb +0 -53
  156. data/test/scanner/test_sets.rb +0 -119
  157. data/test/scanner/test_types.rb +0 -35
  158. data/test/scanner/test_unicode_blocks.rb +0 -30
  159. data/test/support/disable_autotest.rb +0 -8
  160. data/test/syntax/test_all.rb +0 -6
  161. data/test/syntax/test_syntax.rb +0 -61
  162. data/test/syntax/test_syntax_token_map.rb +0 -25
  163. data/test/syntax/versions/test_1.8.rb +0 -55
  164. data/test/syntax/versions/test_1.9.1.rb +0 -36
  165. data/test/syntax/versions/test_1.9.3.rb +0 -32
  166. data/test/syntax/versions/test_2.0.0.rb +0 -37
  167. data/test/syntax/versions/test_2.2.0.rb +0 -32
  168. data/test/syntax/versions/test_aliases.rb +0 -129
  169. data/test/syntax/versions/test_all.rb +0 -5
  170. data/test/test_all.rb +0 -5
  171. data/test/token/test_all.rb +0 -2
  172. data/test/token/test_token.rb +0 -107
@@ -1,36 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_1 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.1'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- escape: [
13
- Escape::Hex + Escape::Octal + Escape::Unicode
14
- ].flatten,
15
-
16
- type: [
17
- CharacterType::Hex
18
- ].flatten,
19
-
20
- quantifier: [
21
- Quantifier::Greedy + Quantifier::Reluctant + Quantifier::Possessive
22
- ].flatten,
23
- },
24
- }
25
-
26
- tests.each do |method, types|
27
- types.each do |type, tokens|
28
- tokens.each do |token|
29
- define_method "test_syntax_V1_9_1_#{method}_#{type}_#{token}" do
30
- assert_equal true, @syntax.implements?(type, token)
31
- end
32
- end
33
- end
34
- end
35
-
36
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV1_9_3 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/1.9.3'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V1_9_3 + UnicodeProperty::Age_V1_9_3
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V1_9_3_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,37 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_0_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.0.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Age_V2_0_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Age_V2_0_0
18
- ].flatten,
19
- },
20
- excludes: {
21
- property: [:newline],
22
- nonproperty: [:newline],
23
- }
24
- }
25
-
26
- tests.each do |method, types|
27
- expected = method == :excludes ? false : true
28
-
29
- types.each do |type, tokens|
30
- tokens.each do |token|
31
- define_method "test_syntax_ruby_V2_0_0_#{method}_#{type}_#{token}" do
32
- assert_equal expected, @syntax.implements?(type, token)
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,32 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxV2_2_0 < Test::Unit::TestCase
4
- include Regexp::Syntax::Token
5
-
6
- def setup
7
- @syntax = Regexp::Syntax.new 'ruby/2.2.0'
8
- end
9
-
10
- tests = {
11
- implements: {
12
- property: [
13
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
14
- ].flatten,
15
-
16
- nonproperty: [
17
- UnicodeProperty::Script_V2_2_0 + UnicodeProperty::Age_V2_2_0
18
- ].flatten,
19
- },
20
- }
21
-
22
- tests.each do |method, types|
23
- types.each do |type, tokens|
24
- tokens.each do |token|
25
- define_method "test_syntax_V2_2_0_#{method}_#{type}_#{token}" do
26
- assert_equal true, @syntax.implements?(type, token)
27
- end
28
- end
29
- end
30
- end
31
-
32
- end
@@ -1,129 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- class TestSyntaxAliases < Test::Unit::TestCase
4
- # 1.8 syntax
5
- def test_syntax_alias_1_8_6
6
- syntax = Regexp::Syntax.new 'ruby/1.8.6'
7
-
8
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
9
- end
10
-
11
- def test_syntax_alias_1_8_alias
12
- syntax = Regexp::Syntax.new 'ruby/1.8'
13
-
14
- assert syntax.is_a?(Regexp::Syntax::V1_8_6)
15
- end
16
-
17
- # 1.9 syntax
18
- def test_syntax_alias_1_9_1
19
- syntax = Regexp::Syntax.new 'ruby/1.9.1'
20
-
21
- assert syntax.is_a?(Regexp::Syntax::V1_9_1)
22
- end
23
-
24
- def test_syntax_alias_1_9_alias
25
- syntax = Regexp::Syntax.new 'ruby/1.9'
26
-
27
- assert syntax.is_a?(Regexp::Syntax::V1_9_3)
28
- end
29
-
30
- # 2.0 syntax
31
- def test_syntax_alias_2_0_0
32
- syntax = Regexp::Syntax.new 'ruby/2.0.0'
33
-
34
- assert syntax.is_a?(Regexp::Syntax::V1_9)
35
- end
36
-
37
- def test_syntax_alias_2_0_alias
38
- syntax = Regexp::Syntax.new 'ruby/2.0'
39
-
40
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
41
- end
42
-
43
- # 2.1 syntax
44
- def test_syntax_alias_2_1_alias
45
- syntax = Regexp::Syntax.new 'ruby/2.1'
46
-
47
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
48
- end
49
-
50
- # 2.2 syntax
51
- def test_syntax_alias_2_2_0
52
- syntax = Regexp::Syntax.new 'ruby/2.2.0'
53
-
54
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
55
- end
56
-
57
- def test_syntax_alias_2_2_10
58
- syntax = Regexp::Syntax.new 'ruby/2.2.10'
59
-
60
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
61
- end
62
-
63
- def test_syntax_alias_2_2_alias
64
- syntax = Regexp::Syntax.new 'ruby/2.2'
65
-
66
- assert syntax.is_a?(Regexp::Syntax::V2_0_0)
67
- end
68
-
69
- # 2.3 syntax
70
- def test_syntax_alias_2_3_0
71
- syntax = Regexp::Syntax.new 'ruby/2.3.0'
72
-
73
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
74
- end
75
-
76
- def test_syntax_alias_2_3
77
- syntax = Regexp::Syntax.new 'ruby/2.3'
78
-
79
- assert syntax.is_a?(Regexp::Syntax::V2_3_0)
80
- end
81
-
82
- # 2.4 syntax
83
- def test_syntax_alias_2_4_0
84
- syntax = Regexp::Syntax.new 'ruby/2.4.0'
85
-
86
- assert syntax.is_a?(Regexp::Syntax::V2_4_0)
87
- end
88
-
89
- def test_syntax_alias_2_4_1
90
- syntax = Regexp::Syntax.new 'ruby/2.4.1'
91
-
92
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
93
- end
94
-
95
- # 2.5 syntax
96
- def test_syntax_alias_2_5_0
97
- syntax = Regexp::Syntax.new 'ruby/2.5.0'
98
-
99
- assert syntax.is_a?(Regexp::Syntax::V2_4_1)
100
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
101
- end
102
-
103
- def test_syntax_alias_2_5
104
- syntax = Regexp::Syntax.new 'ruby/2.5'
105
-
106
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
107
- end
108
-
109
- # 2.6 syntax
110
- def test_syntax_alias_2_6_0
111
- syntax = Regexp::Syntax.new 'ruby/2.6.0'
112
-
113
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
114
- assert syntax.is_a?(Regexp::Syntax::V2_6_0)
115
- end
116
-
117
- def test_syntax_alias_2_6
118
- syntax = Regexp::Syntax.new 'ruby/2.6'
119
-
120
- assert syntax.is_a?(Regexp::Syntax::V2_5_0)
121
- end
122
-
123
- def test_future_alias_warning
124
- _, stderr_output = capture_output { Regexp::Syntax.new 'ruby/5.0' }
125
-
126
- assert_match(/This library .* but you are running .* \(feature set of .*\)/,
127
- stderr_output)
128
- end
129
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../../../helpers", __FILE__)
2
-
3
- %w{aliases 1.8 1.9.1 1.9.3 2.0.0 2.2.0}.each do|tc|
4
- require File.expand_path("../test_#{tc}", __FILE__)
5
- end
@@ -1,5 +0,0 @@
1
- require File.expand_path("../helpers", __FILE__)
2
-
3
- %w{ scanner syntax token lexer parser expression }.each do |subject|
4
- require File.expand_path("../#{subject}/test_all", __FILE__)
5
- end
@@ -1,2 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
- require File.expand_path("../test_token", __FILE__)
@@ -1,107 +0,0 @@
1
- require File.expand_path("../../helpers", __FILE__)
2
-
3
- class Token < Test::Unit::TestCase
4
-
5
- def test_token_offset
6
- regexp = /ab?cd/
7
- tokens = RL.lex(regexp)
8
-
9
- assert_equal 'b', tokens[1].text
10
- assert_equal [1, 2], tokens[1].offset
11
-
12
- assert_equal '?', tokens[2].text
13
- assert_equal [2, 3], tokens[2].offset
14
-
15
- assert_equal 'cd', tokens[3].text
16
- assert_equal [3, 5], tokens[3].offset
17
- end
18
-
19
- def test_token_length
20
- regexp = /abc?def/
21
- tokens = RL.lex(regexp)
22
-
23
- assert_equal 'ab', tokens[0].text
24
- assert_equal 2, tokens[0].length
25
-
26
- assert_equal 'c', tokens[1].text
27
- assert_equal 1, tokens[1].length
28
-
29
- assert_equal '?', tokens[2].text
30
- assert_equal 1, tokens[2].length
31
-
32
- assert_equal 'def', tokens[3].text
33
- assert_equal 3, tokens[3].length
34
- end
35
-
36
- def test_token_to_h
37
- regexp = /abc?def/
38
- tokens = RL.lex(regexp)
39
-
40
- assert_equal 'ab', tokens[0].text
41
- assert_equal({
42
- :type => :literal,
43
- :token => :literal,
44
- :text => 'ab',
45
- :ts => 0,
46
- :te => 2,
47
- :level => 0,
48
- :set_level => 0,
49
- :conditional_level => 0
50
- }, tokens[0].to_h
51
- )
52
-
53
- assert_equal '?', tokens[2].text
54
- assert_equal({
55
- :type => :quantifier,
56
- :token => :zero_or_one,
57
- :text => '?',
58
- :ts => 3,
59
- :te => 4,
60
- :level => 0,
61
- :set_level => 0,
62
- :conditional_level => 0
63
- }, tokens[2].to_h
64
- )
65
- end
66
-
67
- def test_token_next
68
- regexp = /a+b?c*d{2,3}/
69
- tokens = RL.lex(regexp)
70
-
71
- a = tokens.first
72
- assert_equal 'a', a.text
73
-
74
- plus = a.next
75
- assert_equal '+', plus.text
76
-
77
- b = plus.next
78
- assert_equal 'b', b.text
79
-
80
- interval = tokens.last
81
- assert_equal '{2,3}', interval.text
82
-
83
- assert_equal nil, interval.next
84
- end
85
-
86
- def test_token_previous
87
- regexp = /a+b?c*d{2,3}/
88
- tokens = RL.lex(regexp)
89
-
90
- interval = tokens.last
91
- assert_equal '{2,3}', interval.text
92
-
93
- d = interval.previous
94
- assert_equal 'd', d.text
95
-
96
- star = d.previous
97
- assert_equal '*', star.text
98
-
99
- c = star.previous
100
- assert_equal 'c', c.text
101
-
102
- a = tokens.first
103
- assert_equal 'a', a.text
104
- assert_equal nil, a.previous
105
- end
106
-
107
- end