fairy 0.6.0 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Makefile +1 -0
- data/bin/fairy +35 -5
- data/ext/extconf.rb +3 -0
- data/ext/fairy.c +180 -0
- data/ext/fairy.h +94 -0
- data/ext/fiber_mon.h +32 -0
- data/ext/fixnum-buffer.c +483 -0
- data/ext/p-group-by.c +529 -0
- data/ext/p-xgroup-by.c +467 -0
- data/ext/simple-hash.c +44 -0
- data/ext/string-buffer.c +286 -0
- data/ext/xmarshaled-queue.c +699 -0
- data/ext/xsized-queue.c +528 -0
- data/ext/xthread.h +65 -0
- data/fairy.gemspec +5 -2
- data/lib/fairy.rb +10 -1
- data/lib/fairy/client/group-by.rb +57 -2
- data/lib/fairy/client/here.rb +2 -1
- data/lib/fairy/controller.rb +25 -4
- data/lib/fairy/master.rb +17 -3
- data/lib/fairy/master/c-basic-group-by.rb +4 -2
- data/lib/fairy/master/c-cat.rb +3 -2
- data/lib/fairy/master/c-direct-product.rb +5 -3
- data/lib/fairy/master/c-filter.rb +5 -3
- data/lib/fairy/master/c-group-by.rb +13 -0
- data/lib/fairy/master/c-junction.rb +3 -2
- data/lib/fairy/master/c-seg-join.rb +3 -1
- data/lib/fairy/master/c-seg-shuffle.rb +3 -2
- data/lib/fairy/master/c-seg-split.rb +1 -1
- data/lib/fairy/master/c-seg-zip.rb +3 -1
- data/lib/fairy/master/c-sort.rb +7 -2
- data/lib/fairy/master/c-wc.rb +5 -3
- data/lib/fairy/node.rb +13 -2
- data/lib/fairy/node/p-barrier.rb +1 -1
- data/lib/fairy/node/p-basic-group-by.rb +22 -12
- data/lib/fairy/node/p-direct-product.rb +4 -2
- data/lib/fairy/node/p-filter.rb +8 -7
- data/lib/fairy/node/p-find.rb +2 -1
- data/lib/fairy/node/p-group-by.rb +17 -6
- data/lib/fairy/node/p-inject.rb +3 -2
- data/lib/fairy/node/p-output-file.rb +1 -1
- data/lib/fairy/node/p-seg-join.rb +2 -1
- data/lib/fairy/node/p-seg-zip.rb +2 -1
- data/lib/fairy/node/p-single-exportable.rb +3 -1
- data/lib/fairy/node/p-sort.rb +4 -2
- data/lib/fairy/node/p-task.rb +1 -1
- data/lib/fairy/node/p-wc.rb +5 -2
- data/lib/fairy/processor.rb +25 -18
- data/lib/fairy/share/block-source.rb +12 -2
- data/lib/fairy/share/conf.rb +35 -5
- data/lib/fairy/share/hash-simple-hash.rb +1 -1
- data/lib/fairy/share/log.rb +11 -4
- data/lib/fairy/share/pool-dictionary.rb +2 -1
- data/lib/fairy/share/port-marshaled-queue.rb +8 -1
- data/lib/fairy/share/port.rb +55 -45
- data/lib/fairy/share/reference.rb +2 -1
- data/lib/fairy/share/varray.rb +3 -1
- data/lib/fairy/share/vfile.rb +4 -2
- data/lib/fairy/version.rb +1 -1
- data/sample/sort.rb +69 -3
- data/spec/fairy8_spec.rb +1 -1
- data/test/testc.rb +380 -2
- data/tools/cap_recipe/Capfile +3 -3
- data/tools/fairy_conf_wizard.rb +375 -0
- data/tools/fairy_perf_graph.rb +15 -3
- data/tools/git-tag +1 -0
- data/tools/log-analysis.rb +59 -11
- metadata +33 -34
- data/ext/simple_hash/extconf.rb +0 -4
- data/ext/simple_hash/simple_hash.c +0 -42
@@ -61,4 +61,17 @@ module Fairy
|
|
61
61
|
# end
|
62
62
|
|
63
63
|
end
|
64
|
+
|
65
|
+
class CXGroupBy<CGroupBy
|
66
|
+
Controller.def_export self
|
67
|
+
def node_class_name
|
68
|
+
"PXGroupBy"
|
69
|
+
end
|
70
|
+
class CPostFilter<CGroupBy::CPostFilter
|
71
|
+
Controller.def_export self
|
72
|
+
def node_class_name
|
73
|
+
"PXGroupBy::PPostFilter"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
64
77
|
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
|
+
require "xthread"
|
5
6
|
|
6
7
|
require "fairy/master/c-io-filter"
|
7
8
|
require "fairy/master/c-inputtable"
|
@@ -37,11 +38,11 @@ module Fairy
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
40
|
-
class PortQueue<DelegateClass(Queue)
|
41
|
+
class PortQueue<DelegateClass(XThread::Queue)
|
41
42
|
include Enumerable
|
42
43
|
|
43
44
|
def initialize
|
44
|
-
super(Queue.new)
|
45
|
+
super(XThread::Queue.new)
|
45
46
|
end
|
46
47
|
|
47
48
|
def each
|
@@ -3,6 +3,8 @@
|
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
5
|
|
6
|
+
require "xthread"
|
7
|
+
|
6
8
|
require "fairy/master/c-io-filter"
|
7
9
|
require "fairy/master/c-inputtable"
|
8
10
|
|
@@ -20,7 +22,7 @@ module Fairy
|
|
20
22
|
@exports = {}
|
21
23
|
@others_status = {}
|
22
24
|
@exports_mutex = Mutex.new
|
23
|
-
@exports_cv = ConditionVariable.new
|
25
|
+
@exports_cv = XThread::ConditionVariable.new
|
24
26
|
|
25
27
|
@key_proc
|
26
28
|
init_key_proc
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
require "delegate"
|
4
|
+
require "xthread"
|
4
5
|
|
5
6
|
require "fairy/master/c-filter"
|
6
7
|
require "fairy/master/c-io-filter"
|
@@ -70,11 +71,11 @@ module Fairy
|
|
70
71
|
end
|
71
72
|
end
|
72
73
|
|
73
|
-
class PortQueue<DelegateClass(Queue)
|
74
|
+
class PortQueue<DelegateClass(XThread::Queue)
|
74
75
|
include Enumerable
|
75
76
|
|
76
77
|
def initialize
|
77
|
-
super(Queue.new)
|
78
|
+
super(XThread::Queue.new)
|
78
79
|
end
|
79
80
|
|
80
81
|
def each
|
@@ -3,6 +3,8 @@
|
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
5
|
|
6
|
+
require "xthread"
|
7
|
+
|
6
8
|
require "fairy/master/c-io-filter"
|
7
9
|
require "fairy/master/c-inputtable"
|
8
10
|
|
@@ -23,7 +25,7 @@ module Fairy
|
|
23
25
|
@exports = []
|
24
26
|
@others_status = {}
|
25
27
|
@exports_mutex = Mutex.new
|
26
|
-
@exports_cv = ConditionVariable.new
|
28
|
+
@exports_cv = XThread::ConditionVariable.new
|
27
29
|
end
|
28
30
|
|
29
31
|
def opt_zip_by_substream?
|
data/lib/fairy/master/c-sort.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
|
+
require "xthread"
|
5
6
|
|
6
7
|
require "fairy/master/c-io-filter"
|
7
8
|
require "fairy/master/c-inputtable"
|
@@ -20,7 +21,7 @@ module Fairy
|
|
20
21
|
|
21
22
|
@pvs = nil
|
22
23
|
@pvs_mutex = Mutex.new
|
23
|
-
@pvs_cv = ConditionVariable.new
|
24
|
+
@pvs_cv = XThread::ConditionVariable.new
|
24
25
|
end
|
25
26
|
|
26
27
|
def get_pvs(buf=nil)
|
@@ -59,12 +60,16 @@ module Fairy
|
|
59
60
|
key_proc = BBlock.new(@block_source, @context, self)
|
60
61
|
end
|
61
62
|
|
62
|
-
sorted = @samplings.flatten(1).
|
63
|
+
sorted = @samplings.flatten(1).map{|e| key_proc.call(e)}.sort_by{|e| e}
|
63
64
|
|
64
65
|
#Log::debugf(self, "%s", sorted.inspect)
|
65
66
|
idxes = (1...no_segment).collect{|i| (sorted.size*i).div(no_segment)}
|
66
67
|
@pvs_mutex.synchronize do
|
67
68
|
@pvs = sorted.values_at(*idxes)
|
69
|
+
sorted.clear
|
70
|
+
sorted = nil
|
71
|
+
@samplings.clear
|
72
|
+
@samplings = nil
|
68
73
|
@pvs_cv.broadcast
|
69
74
|
end
|
70
75
|
end
|
data/lib/fairy/master/c-wc.rb
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
5
|
|
6
|
+
require "xthread"
|
7
|
+
|
6
8
|
require "fairy/master/c-io-filter"
|
7
9
|
require "fairy/master/c-input"
|
8
10
|
require "fairy/share/vfile"
|
@@ -27,10 +29,10 @@ module Fairy
|
|
27
29
|
# key -> [export, ...]
|
28
30
|
@exports = {}
|
29
31
|
@exports_mutex = Mutex.new
|
30
|
-
@exports_cv = ConditionVariable.new
|
32
|
+
@exports_cv = XThread::ConditionVariable.new
|
31
33
|
|
32
|
-
# @pre_exports_queue = Queue.new
|
33
|
-
@exports_queue = Queue.new
|
34
|
+
# @pre_exports_queue = XThread::Queue.new
|
35
|
+
@exports_queue = XThread::Queue.new
|
34
36
|
|
35
37
|
@each_export_by_thread = nil
|
36
38
|
@each_export_by_thread_mutex = Mutex.new
|
data/lib/fairy/node.rb
CHANGED
@@ -7,11 +7,15 @@
|
|
7
7
|
|
8
8
|
require "timeout"
|
9
9
|
|
10
|
+
require "xthread"
|
11
|
+
require "fiber-mon"
|
12
|
+
|
10
13
|
require "deep-connect"
|
11
14
|
|
12
15
|
require "fairy/version"
|
13
16
|
require "fairy/share/conf"
|
14
17
|
|
18
|
+
|
15
19
|
#DeepConnect::Organizer.immutable_classes.push Array
|
16
20
|
|
17
21
|
module Fairy
|
@@ -27,11 +31,11 @@ module Fairy
|
|
27
31
|
|
28
32
|
@processors = []
|
29
33
|
@processors_mutex = Mutex.new
|
30
|
-
@processors_cv = ConditionVariable.new
|
34
|
+
@processors_cv = XThread::ConditionVariable.new
|
31
35
|
|
32
36
|
@active_processors = {}
|
33
37
|
@active_processors_mutex = Mutex.new
|
34
|
-
@active_processors_cv = ConditionVariable.new
|
38
|
+
@active_processors_cv = XThread::ConditionVariable.new
|
35
39
|
end
|
36
40
|
|
37
41
|
attr_accessor :id
|
@@ -67,6 +71,13 @@ module Fairy
|
|
67
71
|
Log::info(self, "\tfairy version: #{Version}")
|
68
72
|
Log::info(self, "\t[Powered BY #{RUBY_DESCRIPTION}]")
|
69
73
|
|
74
|
+
begin
|
75
|
+
require "fairy.so"
|
76
|
+
Log::warn self, "\t Load fairy.so"
|
77
|
+
rescue LoadError
|
78
|
+
Log::warn self, "Can't load fairy.so. Can't use this feature"
|
79
|
+
end
|
80
|
+
|
70
81
|
@master.register_node(self)
|
71
82
|
end
|
72
83
|
|
data/lib/fairy/node/p-barrier.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
|
+
require "xthread"
|
5
6
|
|
6
7
|
require "fairy/node/p-io-filter"
|
7
8
|
|
@@ -18,13 +19,25 @@ module Fairy
|
|
18
19
|
@block_source = block_source
|
19
20
|
|
20
21
|
@exports = {}
|
21
|
-
@exports_queue = Queue.new
|
22
|
+
@exports_queue = XThread::Queue.new
|
22
23
|
|
23
24
|
@counter = {}
|
24
25
|
|
25
26
|
#start_watch_exports
|
26
27
|
end
|
27
28
|
|
29
|
+
def init_key_proc
|
30
|
+
hash_opt = @opts[:grouping_optimize]
|
31
|
+
hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
|
32
|
+
|
33
|
+
if hash_opt
|
34
|
+
@key_proc = eval("proc{#{@block_source.source}}", @context.binding)
|
35
|
+
else
|
36
|
+
@key_proc = BBlock.new(@block_source, @context, self)
|
37
|
+
end
|
38
|
+
@key_proc
|
39
|
+
end
|
40
|
+
|
28
41
|
def add_export(key, export)
|
29
42
|
@exports[key] = export
|
30
43
|
# @exports_queue.push [key, export]
|
@@ -36,14 +49,7 @@ module Fairy
|
|
36
49
|
Log::debug(self, "START_EXPORT")
|
37
50
|
|
38
51
|
start do
|
39
|
-
|
40
|
-
hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
|
41
|
-
|
42
|
-
if hash_opt
|
43
|
-
@key_proc = eval("proc{#{@block_source.source}}", @context.binding)
|
44
|
-
else
|
45
|
-
@key_proc = BBlock.new(@block_source, @context, self)
|
46
|
-
end
|
52
|
+
init_key_proc
|
47
53
|
|
48
54
|
policy = @opts[:postqueuing_policy]
|
49
55
|
begin
|
@@ -98,7 +104,9 @@ module Fairy
|
|
98
104
|
Log::debug(self, "G3")
|
99
105
|
@exports.each_pair do |key, export|
|
100
106
|
Log::debug(self, "G3.WAIT #{key}")
|
101
|
-
|
107
|
+
@terminate_mon.synchronize do
|
108
|
+
export.fib_wait_finish(@wait_cv)
|
109
|
+
end
|
102
110
|
end
|
103
111
|
Log::debug(self, "G4")
|
104
112
|
self.status = ST_EXPORT_FINISH
|
@@ -121,7 +129,7 @@ module Fairy
|
|
121
129
|
@key_proc = BBlock.new(@block_source, @context, self)
|
122
130
|
|
123
131
|
@exports = {}
|
124
|
-
@exports_queue = Queue.new
|
132
|
+
@exports_queue = XThread::Queue.new
|
125
133
|
|
126
134
|
# start_watch_exports
|
127
135
|
end
|
@@ -227,7 +235,9 @@ Log::debug(self, "G3")
|
|
227
235
|
@exports.each_pair do |key, export|
|
228
236
|
next unless export
|
229
237
|
Log::debug(self, "G4.WAIT #{key}")
|
230
|
-
|
238
|
+
@terminate_mon.synchronize do
|
239
|
+
export.fib_wait_finish(@wait_cv)
|
240
|
+
end
|
231
241
|
end
|
232
242
|
Log::debug(self, "G5")
|
233
243
|
self.status = ST_EXPORT_FINISH
|
@@ -3,6 +3,8 @@
|
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
5
|
|
6
|
+
require "xthread"
|
7
|
+
|
6
8
|
require "fairy/node/p-io-filter"
|
7
9
|
require "fairy/node/p-single-exportable"
|
8
10
|
|
@@ -17,7 +19,7 @@ module Fairy
|
|
17
19
|
|
18
20
|
@exports = nil
|
19
21
|
@exports_mutex = Mutex.new
|
20
|
-
@exports_cv = ConditionVariable.new
|
22
|
+
@exports_cv = XThread::ConditionVariable.new
|
21
23
|
end
|
22
24
|
|
23
25
|
def input=(input)
|
@@ -86,7 +88,7 @@ module Fairy
|
|
86
88
|
|
87
89
|
@other_imports = nil
|
88
90
|
@other_imports_mutex = Mutex.new
|
89
|
-
@other_imports_cv = ConditionVariable.new
|
91
|
+
@other_imports_cv = XThread::ConditionVariable.new
|
90
92
|
end
|
91
93
|
|
92
94
|
def input=(input)
|
data/lib/fairy/node/p-filter.rb
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
#
|
5
5
|
|
6
6
|
require "thread"
|
7
|
+
require "xthread"
|
7
8
|
|
8
9
|
require "fairy/node/p-task"
|
9
10
|
|
@@ -59,18 +60,18 @@ module Fairy
|
|
59
60
|
|
60
61
|
@no = nil
|
61
62
|
@no_mutex = Mutex.new
|
62
|
-
@no_cv = ConditionVariable.new
|
63
|
+
@no_cv = XThread::ConditionVariable.new
|
63
64
|
|
64
65
|
@key = nil
|
65
66
|
@key_mutex = Mutex.new
|
66
|
-
@key_cv = ConditionVariable.new
|
67
|
+
@key_cv = XThread::ConditionVariable.new
|
67
68
|
|
68
69
|
@status = ST_INIT
|
69
|
-
@status_mon = processor.njob_mon
|
70
|
+
@status_mon = processor.njob_mon.new_mon
|
70
71
|
@status_cv = @status_mon.new_cv
|
71
72
|
notice_status(@status)
|
72
73
|
|
73
|
-
@terminate_mon = processor.njob_mon
|
74
|
+
@terminate_mon = processor.njob_mon.new_mon
|
74
75
|
|
75
76
|
@in_each = nil
|
76
77
|
@in_each_mutex = Mutex.new
|
@@ -151,7 +152,7 @@ module Fairy
|
|
151
152
|
end
|
152
153
|
|
153
154
|
@main_thread = nil
|
154
|
-
|
155
|
+
processor.njob_mon.entry terminate_proc
|
155
156
|
Log::info self, "FINISH PROCESSING: #{self.class}"
|
156
157
|
end
|
157
158
|
rescue Exception
|
@@ -282,7 +283,7 @@ module Fairy
|
|
282
283
|
# 初期状態通知
|
283
284
|
notice_status(@status)
|
284
285
|
|
285
|
-
|
286
|
+
processor.njob_mon.entry do
|
286
287
|
@status_mon.synchronize do
|
287
288
|
old_status = nil
|
288
289
|
loop do
|
@@ -297,7 +298,7 @@ module Fairy
|
|
297
298
|
end
|
298
299
|
|
299
300
|
def notice_status(st)
|
300
|
-
|
301
|
+
processor.njob_mon.entry do
|
301
302
|
@bjob.update_status(self, st)
|
302
303
|
@ntask.update_status(self, st)
|
303
304
|
end
|
data/lib/fairy/node/p-find.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
|
+
require "xthread"
|
5
6
|
|
6
7
|
require "fairy/node/p-io-filter"
|
7
8
|
require "fairy/node/p-single-exportable"
|
@@ -65,7 +66,7 @@ module Fairy
|
|
65
66
|
|
66
67
|
@value = :__FAIRY_NO_VALUE__
|
67
68
|
@value_mutex = Mutex.new
|
68
|
-
@value_cv = ConditionVariable.new
|
69
|
+
@value_cv = XThread::ConditionVariable.new
|
69
70
|
end
|
70
71
|
|
71
72
|
def input=(input)
|
@@ -2,6 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# Copyright (C) 2007-2010 Rakuten, Inc.
|
4
4
|
#
|
5
|
+
require "xthread"
|
5
6
|
|
6
7
|
require "fairy/node/p-io-filter"
|
7
8
|
require "fairy/node/p-basic-group-by"
|
@@ -60,6 +61,15 @@ module Fairy
|
|
60
61
|
end
|
61
62
|
end
|
62
63
|
|
64
|
+
def init_key_proc
|
65
|
+
if @hash_optimize
|
66
|
+
@hash_proc = eval("proc{#{@block_source.source}}")
|
67
|
+
else
|
68
|
+
@hash_proc = BBlock.new(@block_source, @context, self)
|
69
|
+
end
|
70
|
+
@hash_proc
|
71
|
+
end
|
72
|
+
|
63
73
|
# def start
|
64
74
|
# super do
|
65
75
|
# @key_value_buffer =
|
@@ -97,11 +107,7 @@ module Fairy
|
|
97
107
|
def basic_each(&block)
|
98
108
|
@key_value_buffer =
|
99
109
|
eval("#{@buffering_policy[:buffering_class]}").new(self, @buffering_policy)
|
100
|
-
|
101
|
-
@hash_proc = eval("proc{#{@block_source.source}}")
|
102
|
-
else
|
103
|
-
@hash_proc = BBlock.new(@block_source, @context, self)
|
104
|
-
end
|
110
|
+
init_key_proc
|
105
111
|
|
106
112
|
@input.each do |e|
|
107
113
|
@key_value_buffer.push(e)
|
@@ -589,7 +595,7 @@ module Fairy
|
|
589
595
|
|
590
596
|
def finish_wait
|
591
597
|
@mx = Mutex.new
|
592
|
-
@cv = ConditionVariable.new
|
598
|
+
@cv = XThread::ConditionVariable.new
|
593
599
|
@mx.synchronize do
|
594
600
|
@cv.wait(@mx)
|
595
601
|
end
|
@@ -1053,6 +1059,11 @@ module Fairy
|
|
1053
1059
|
|
1054
1060
|
read_buffer
|
1055
1061
|
@key = @njob.hash_key(@cache.first)
|
1062
|
+
#Log::debug(self, "READ_KEY #{@key}");
|
1063
|
+
#unless @key
|
1064
|
+
# Log::debug(self, "READ_BUFFER #{@cache}");
|
1065
|
+
#end
|
1066
|
+
|
1056
1067
|
end
|
1057
1068
|
|
1058
1069
|
def_delegator :@io, :open
|