regextest 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +3 -0
  5. data/Gemfile +4 -0
  6. data/LICENSE.txt +25 -0
  7. data/README.md +88 -0
  8. data/Rakefile +55 -0
  9. data/bin/console +14 -0
  10. data/bin/regextest +4 -0
  11. data/bin/setup +7 -0
  12. data/contrib/Onigmo/RE.txt +522 -0
  13. data/contrib/Onigmo/UnicodeProps.txt +728 -0
  14. data/contrib/Onigmo/testpy.py +1319 -0
  15. data/contrib/unicode/Blocks.txt +298 -0
  16. data/contrib/unicode/CaseFolding.txt +1414 -0
  17. data/contrib/unicode/DerivedAge.txt +1538 -0
  18. data/contrib/unicode/DerivedCoreProperties.txt +11029 -0
  19. data/contrib/unicode/PropList.txt +1525 -0
  20. data/contrib/unicode/PropertyAliases.txt +193 -0
  21. data/contrib/unicode/PropertyValueAliases.txt +1420 -0
  22. data/contrib/unicode/README.txt +25 -0
  23. data/contrib/unicode/Scripts.txt +2539 -0
  24. data/contrib/unicode/UnicodeData.txt +29215 -0
  25. data/lib/pre-case-folding.rb +101 -0
  26. data/lib/pre-posix-char-class.rb +150 -0
  27. data/lib/pre-unicode.rb +116 -0
  28. data/lib/regextest.rb +268 -0
  29. data/lib/regextest/back.rb +58 -0
  30. data/lib/regextest/back/element.rb +151 -0
  31. data/lib/regextest/back/main.rb +356 -0
  32. data/lib/regextest/back/result.rb +498 -0
  33. data/lib/regextest/back/test-case.rb +268 -0
  34. data/lib/regextest/back/work-thread.rb +119 -0
  35. data/lib/regextest/common.rb +63 -0
  36. data/lib/regextest/front.rb +60 -0
  37. data/lib/regextest/front/anchor.rb +45 -0
  38. data/lib/regextest/front/back-refer.rb +120 -0
  39. data/lib/regextest/front/bracket-parser.rb +400 -0
  40. data/lib/regextest/front/bracket-parser.y +117 -0
  41. data/lib/regextest/front/bracket-scanner.rb +124 -0
  42. data/lib/regextest/front/bracket.rb +64 -0
  43. data/lib/regextest/front/builtin-functions.rb +31 -0
  44. data/lib/regextest/front/case-folding.rb +18 -0
  45. data/lib/regextest/front/char-class.rb +243 -0
  46. data/lib/regextest/front/empty.rb +43 -0
  47. data/lib/regextest/front/letter.rb +327 -0
  48. data/lib/regextest/front/manage-parentheses.rb +74 -0
  49. data/lib/regextest/front/parenthesis.rb +153 -0
  50. data/lib/regextest/front/parser.rb +1366 -0
  51. data/lib/regextest/front/parser.y +271 -0
  52. data/lib/regextest/front/range.rb +60 -0
  53. data/lib/regextest/front/repeat.rb +90 -0
  54. data/lib/regextest/front/repeatable.rb +77 -0
  55. data/lib/regextest/front/scanner.rb +187 -0
  56. data/lib/regextest/front/selectable.rb +65 -0
  57. data/lib/regextest/front/sequence.rb +73 -0
  58. data/lib/regextest/front/unicode.rb +1272 -0
  59. data/lib/regextest/regex-option.rb +144 -0
  60. data/lib/regextest/regexp.rb +44 -0
  61. data/lib/regextest/version.rb +5 -0
  62. data/lib/tst-reg-test.rb +159 -0
  63. data/regextest.gemspec +26 -0
  64. metadata +162 -0
@@ -0,0 +1,268 @@
1
+ # encoding: utf-8
2
+
3
+ # Copyright (C) 2016 Mikio Ikoma
4
+
5
+ # NOT USED AT PRESENT
6
+
7
+
8
+ # ZDDライブラリ
9
+ require "nysol/zdd"
10
+
11
+ # ZDDライブラリを使って制約条件を満たすテストケースを生成
12
+ class Regextest::Back::TestCase
13
+
14
+ def initialize(json_obj, name_hash)
15
+ @json_obj = json_obj
16
+ @name_hash = name_hash
17
+ @selectables = {}
18
+ @selectable_num = 0
19
+ @constraints = []
20
+ @paren_hash = {}
21
+ @test_cases = nil
22
+ @called_num = 0
23
+
24
+ # 解析木をサーチして、選択可能な要素を列挙
25
+ seek_selectable(@json_obj, [])
26
+ enum_selectables
27
+
28
+ # ZDDを使って制約条件からテストケースを生成
29
+ solve_constraints
30
+ end
31
+
32
+ # 解析木をサーチして、選択可能な要素を列挙する
33
+ def seek_selectable(target, stack)
34
+ case target["type"]
35
+ when "LEX_SEQ"
36
+ target["value"].map{|elem| seek_selectable(elem, stack)}
37
+ when "LEX_SELECT"
38
+ if(target["value"].size > 1)
39
+ add_selectable(target, target["value"].size, stack)
40
+ target["value"].each_with_index do |elem, i|
41
+ stack.push "#{target["id"]}_#{i}"
42
+ seek_selectable(elem, stack)
43
+ stack.pop
44
+ end
45
+ else
46
+ # 一つしかない場合
47
+ seek_selectable(target["value"][0], stack)
48
+ end
49
+ when "LEX_PAREN"
50
+ # 制約条件があるカッコは、その情報を記憶しておく(後方参照用)
51
+ if(stack.size > 0)
52
+ puts "paren id #{target["id"]} stack #{stack[-1]}"
53
+ add_paren(target["id"], stack[-1])
54
+ end
55
+ seek_selectable(target["value"], stack)
56
+ when "LEX_BRACKET", "LEX_SIMPLIFIED_CLASS", "LEX_ANY_LETTER", "LEX_POSIX_CHAR_CLASS"
57
+ seek_selectable(target["value"], stack)
58
+ when "LEX_REPEAT"
59
+ if(target["max_repeat"] > target["min_repeat"]+1)
60
+ add_selectable(target, 3, stack)
61
+ elsif(target["max_repeat"] == target["min_repeat"]+1)
62
+ add_selectable(target, 2, stack)
63
+ elsif(target["max_repeat"] == 0)
64
+ # (foo){0}のパターン
65
+ add_selectable(target, 1, stack)
66
+ end
67
+
68
+ # 繰り返し数の最小がゼロの場合は省略される可能性があるのでスタックを積む
69
+ if(target["min_repeat"] == 0)
70
+ stack.push "~#{target["id"]}_0"
71
+ seek_selectable(target["value"], stack)
72
+ stack.pop
73
+ else
74
+ seek_selectable(target["value"], stack)
75
+ end
76
+ when "LEX_RANGE"
77
+ # add_selectable(target, stack)
78
+ when "LEX_BACK_REFER", "LEX_NAMED_REFER"
79
+ # constraintsの登録
80
+ referred = @name_hash[target["refer_name"]]["id"]
81
+ raise "Internal error, not found referred parenthesis(#{target["refer_name"]})" if(!referred)
82
+ puts "referred = #{referred}, stack=#{stack}"
83
+ if(stack.size > 0)
84
+ # その後方参照が参照される場合は、参照先も存在している必要がある。
85
+ add_constraint(stack[-1], referred)
86
+ else
87
+ # 無条件に参照先が存在している必要がある
88
+ add_constraint(nil, referred)
89
+ end
90
+ when "LEX_NAMED_GENERATE", "LEX_CHAR", "LEX_UNICODE_CLASS"
91
+ # 処理は不要?
92
+ else
93
+ raise "#{target["type"]} not implemented (from seek_selectable routine)"
94
+ end
95
+ end
96
+
97
+ # カッコの情報を追加
98
+ def add_paren(paren_id, selectable_id)
99
+ @paren_hash[paren_id] = convert_selectable_id(selectable_id)
100
+ end
101
+
102
+ # 生成したテストケースから一つのテストを得る
103
+ def get_test_case
104
+ if(@test_cases)
105
+ result = @test_cases[@called_num].dup
106
+ @called_num += 1
107
+
108
+ # 一巡した場合、シャッフルして最初から
109
+ if(@called_num == @test_cases.size)
110
+ @called_num = 0
111
+ @test_cases.shuffle!
112
+ end
113
+ else
114
+ result = nil
115
+ end
116
+ result
117
+ end
118
+
119
+ # 制約条件の追加
120
+ def add_constraint(requires_obj, then_obj, else_obj = nil)
121
+ @constraints.push [requires_obj, then_obj, else_obj]
122
+ end
123
+
124
+ # 制約条件のZDD化
125
+ def refine_constraints
126
+ # pp @constraints
127
+ @constraints.map! do | constraint |
128
+ constraint.map do | elem |
129
+ if(String === elem)
130
+ key = elem.split("_")[0]
131
+ if(@selectables[key])
132
+ ZDD.itemset(elem)
133
+ elsif(@paren_hash[elem])
134
+ convert_to_zdd(@paren_hash[elem])
135
+ else
136
+ # pp @selectables
137
+ # raise "Internal error: cannot convert from #{elem} to selectable"
138
+ nil
139
+ end
140
+ else
141
+ elem
142
+ end
143
+ end
144
+ end
145
+ end
146
+
147
+ # 制約条件のZDD化(行列表現をZDDの積和形式に変換)
148
+ def convert_to_zdd(elem)
149
+ if Array === elem
150
+ work = ZDD.constant(0)
151
+ elem.each do | elem2 |
152
+ work += ZDD.itemset(elem2)
153
+ end
154
+ work
155
+ else
156
+ ZDD.itemset(elem)
157
+ end
158
+ end
159
+
160
+ # ~否定の演算子を変換
161
+ def convert_selectable_id(selectable_id)
162
+ if(md = selectable_id.match(/^(~)?(.+?)_(\d+)$/))
163
+ if(md[1])
164
+ # ~付きは、それ以外のもの
165
+ key = md[2]
166
+ num = md[3].to_i
167
+ selectable_ids = (1...@selectables[key][:level]).to_a.map{|num| "#{key}_#{num}"}
168
+ else
169
+ selectable_id
170
+ end
171
+ else
172
+ raise "Internal error: invalid selectable_id #{selectable_id}"
173
+ end
174
+ end
175
+
176
+ # 選択可能なエントリの列挙
177
+ def enum_selectables
178
+ @selectables.each do | key, value |
179
+ target = value[:target]
180
+ puts "name: #{key}, type: #{target["type"]}, " +
181
+ "requires: #{value[:requires]}, " +
182
+ "level_num: #{value[:level]}"
183
+ end
184
+ end
185
+
186
+ # 選択可能な要素の登録
187
+ def add_selectable(target, level_num, stack)
188
+ name = target["id"]
189
+ raise "Error: internal error, name not found" if(!name)
190
+ @selectable_num += 1
191
+ @selectables[name] = {:target => target, :requires => stack[-1], :level => level_num}
192
+ name
193
+ end
194
+
195
+ # 制約条件の簡略化および適用
196
+ def solve_constraints
197
+
198
+ # 全ての組合せを求める。
199
+ test_set = ZDD.constant(1)
200
+ @selectables.each do | key, value |
201
+ # 通常の値の設定
202
+ params = ZDD.constant(0)
203
+ value[:level].times do | i |
204
+ value_name = "#{key}_#{i}"
205
+ params += ZDD.itemset(value_name)
206
+ end
207
+
208
+ # 制約がある場合の設定
209
+ if(value[:requires])
210
+ # 省略値の設定
211
+ empty_value = ZDD.itemset("#{key}__")
212
+ # 依存関係のある値があるときは、通常の値。無いときは省略値
213
+ puts value[:requires]
214
+ require_id = convert_selectable_id(value[:requires])
215
+ add_constraint(convert_to_zdd(require_id), params, empty_value)
216
+ params += empty_value
217
+ end
218
+
219
+ # まず、全ての組わせを求める
220
+ test_set *= params
221
+ end
222
+ test_set.show if(test_set.count < 256)
223
+ puts "whole test_set is #{test_set.count}"
224
+
225
+ # 制約でテストを減らす
226
+ refine_constraints
227
+ # pp @constraints
228
+ @constraints.each do | constraint |
229
+ if(!constraint[1] && !constraint[2])
230
+ # 何もしない
231
+ elsif(!constraint[0])
232
+ # 無条件制約
233
+ test_set = test_set.restrict(constraint[1])
234
+ elsif(!constraint[2])
235
+ # if then の制約
236
+ test_set = test_set.restrict(constraint[0]).
237
+ iif(test_set.restrict(constraint[1]), test_set)
238
+ else
239
+ # if then elseの制約
240
+ test_set = test_set.restrict(constraint[0]).
241
+ iif(test_set.restrict(constraint[1]), test_set.restrict(constraint[2]))
242
+ end
243
+ end
244
+ test_set.show if(test_set.count < 256)
245
+ puts "whole test_set is #{test_set.count}"
246
+
247
+ if(test_set.same?(ZDD.constant(1)))
248
+ puts "no selectable element"
249
+ else
250
+ @test_cases = test_set.to_a.shuffle.map do | a_test |
251
+ test_case = {}
252
+ a_test.split(/\s+/).each do | item |
253
+ elems = item.split("_")
254
+ test_case[elems[0]] = (elems[1])?(elems[1].to_i):nil
255
+ end
256
+ test_case
257
+ end
258
+ end
259
+ pp @test_cases if @test_cases
260
+ end
261
+ end
262
+
263
+
264
+ # Test suite (execute when this file is specified in command line)
265
+ if __FILE__ == $0
266
+
267
+ end
268
+
@@ -0,0 +1,119 @@
1
+ # encoding: utf-8
2
+
3
+ # Copyright (C) 2016 Mikio Ikoma
4
+
5
+ # NOT USED AT PRESENT
6
+
7
+ require "pp"
8
+ require "thread"
9
+
10
+ class Regextest::Back::WorkThread
11
+ def initialize(thread_id, initial_data = nil)
12
+ @thread_id = thread_id
13
+ @data = initial_data
14
+ @r_queue = Queue.new
15
+ @s_queue = Queue.new
16
+ @proc = nil
17
+ end
18
+
19
+ attr_reader :thread_id
20
+
21
+ # Set procedure
22
+ def run(&proc)
23
+ @proc = proc
24
+ @thread = Thread.new {
25
+ Thread.abort_on_exception = true
26
+ @proc.call(@data)
27
+ }
28
+ end
29
+
30
+ # Request data to child thread (executed in parent thread)
31
+ def request(data)
32
+ @r_queue.push(data)
33
+ end
34
+
35
+ # Indicate data from parent thread (executed in child thread)
36
+ def indicate
37
+ @r_queue.pop
38
+ end
39
+
40
+ # Respond data to parent thread (executed in child thread)
41
+ def respond(data)
42
+ @s_queue.push(data)
43
+ end
44
+
45
+ # Confirm data from child thread (executed in parent thread)
46
+ def confirm
47
+ @s_queue.pop
48
+ end
49
+
50
+ # Wait to start (executed in child thread)
51
+ def wait
52
+ data = indicate
53
+ raise ("invalid wait. received data is #{data}") if(data != :THR_CMD_START)
54
+ end
55
+
56
+ # Start child's thread (executed in parent thread)
57
+ def start
58
+ request(:THR_CMD_START)
59
+ end
60
+
61
+ # Terminate child's thread (executed in parent thread)
62
+ def terminate
63
+ request(:THR_CMD_TERMINATE)
64
+ @thread.join
65
+ end
66
+
67
+ # Exit thread (executed in child thread)
68
+ def exit
69
+ respond(:THR_CMD_EXIT)
70
+ end
71
+
72
+ def exit?(data)
73
+ data == :THR_CMD_EXIT
74
+ end
75
+ end
76
+
77
+ # Test suite (execute when this file is specified in command line)
78
+ if __FILE__ == $0
79
+ Thread.abort_on_exception = true
80
+
81
+ thread_num = 3
82
+ threads = []
83
+ data = [[[0,1],[0,1],[0,2]],[[0,1,2],[0,1],[0,1]], [[1],[1],[0]]]
84
+ thread_num.times do | i |
85
+ thread_obj = Regextest::Back::WorkThread.new(i, data[i])
86
+ thread_obj.run { | param |
87
+ while((command = thread_obj.indicate) != :THR_CMD_TERMINATE)
88
+ puts "get command #{command}"
89
+ reply = param.shift
90
+ thread_obj.respond(reply)
91
+ end
92
+ puts "terminate thread"
93
+ }
94
+ threads << thread_obj
95
+ end
96
+ pp threads
97
+ data[0].size.times do
98
+ result = nil
99
+ threads.each do | thread |
100
+ thread.request("do something #{thread}")
101
+ reply = thread.confirm
102
+ if(result == nil)
103
+ result = reply
104
+ else
105
+ result &= reply
106
+ end
107
+ end
108
+ puts "result is #{result}"
109
+ end
110
+
111
+ threads.each do | thread |
112
+ thread.terminate
113
+ end
114
+
115
+ pp threads
116
+
117
+ end
118
+
119
+
@@ -0,0 +1,63 @@
1
+ # encoding: utf-8
2
+
3
+ # Copyright (C) 2016 Mikio Ikoma
4
+
5
+ # Common part of regextest
6
+ module Regextest::Common
7
+ # Analyzing options
8
+ @@parse_options = nil
9
+ @@rand_called = false
10
+
11
+ # environment variables
12
+ TstConstRetryMax = (ENV['REGEXTEST_MAX_RETRY'])?(ENV['REGEXTEST_MAX_RETRY'].to_i):5
13
+ TstConstRepeatMax = (ENV['REGEXTEST_MAX_REPEAT'])?(ENV['REGEXTEST_MAX_REPEAT'].to_i):32
14
+ TstConstRecursionMax = (ENV['REGEXTEST_MAX_RECURSION'])?(ENV['REGEXTEST_MAX_RECURSION'].to_i):32
15
+ TstConstDebug = (ENV['REGEXTEST_DEBUG'])?true:false
16
+ TstConstTimeout = (ENV['REGEXTEST_TIMEOUT'])?(ENV['REGEXTEST_TIMEOUT'].to_f):1.0 # default is 1 second
17
+
18
+ # whole character set if unicode mode. specify as 'ascii|kana', 'ascii|han|kana', etc.
19
+ TstConstUnicodeCharSet = (ENV['REGEXTEST_UNICODE_CHAR_SET'] || "ascii|katakana|hiragana").downcase
20
+
21
+ # Log
22
+ def TstLog(msg)
23
+ # if(!defined? Rails) # not output debug message when rails env (even if development mode)
24
+ if TstConstDebug
25
+ warn msg
26
+ end
27
+ # end
28
+ end
29
+
30
+ # Randomize
31
+ def TstRand(num)
32
+ @@rand_called = true
33
+ rand(num)
34
+ end
35
+
36
+ # Shuffle
37
+ def TstShuffle(array)
38
+ @@rand_called = true
39
+ array.shuffle
40
+ end
41
+
42
+ # reset random_called
43
+ def reset_random_called
44
+ @@rand_called = false
45
+ end
46
+
47
+ # is_random?
48
+ def is_random?
49
+ @@rand_called
50
+ end
51
+
52
+ # Pretty print of matched data object
53
+ def TstMdPrint(md)
54
+ # coloring if tty && (!windows)
55
+ if $stdout.tty? && !RUBY_PLATFORM.downcase.match(/mswin(?!ce)|mingw/)
56
+ "#{md.pre_match.inspect[1..-2]}\e[36m#{md.to_a[0].inspect[1..-2]}\e[0m#{md.post_match.inspect[1..-2]}"
57
+ else
58
+ "#{md.pre_match.inspect[1..-2]} #{md.to_a[0].inspect[1..-2]} #{md.post_match.inspect[1..-2]}"
59
+ end
60
+ end
61
+
62
+ end
63
+