fairy 0.6.0 → 0.6.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. data/Makefile +1 -0
  2. data/bin/fairy +35 -5
  3. data/ext/extconf.rb +3 -0
  4. data/ext/fairy.c +180 -0
  5. data/ext/fairy.h +94 -0
  6. data/ext/fiber_mon.h +32 -0
  7. data/ext/fixnum-buffer.c +483 -0
  8. data/ext/p-group-by.c +529 -0
  9. data/ext/p-xgroup-by.c +467 -0
  10. data/ext/simple-hash.c +44 -0
  11. data/ext/string-buffer.c +286 -0
  12. data/ext/xmarshaled-queue.c +699 -0
  13. data/ext/xsized-queue.c +528 -0
  14. data/ext/xthread.h +65 -0
  15. data/fairy.gemspec +5 -2
  16. data/lib/fairy.rb +10 -1
  17. data/lib/fairy/client/group-by.rb +57 -2
  18. data/lib/fairy/client/here.rb +2 -1
  19. data/lib/fairy/controller.rb +25 -4
  20. data/lib/fairy/master.rb +17 -3
  21. data/lib/fairy/master/c-basic-group-by.rb +4 -2
  22. data/lib/fairy/master/c-cat.rb +3 -2
  23. data/lib/fairy/master/c-direct-product.rb +5 -3
  24. data/lib/fairy/master/c-filter.rb +5 -3
  25. data/lib/fairy/master/c-group-by.rb +13 -0
  26. data/lib/fairy/master/c-junction.rb +3 -2
  27. data/lib/fairy/master/c-seg-join.rb +3 -1
  28. data/lib/fairy/master/c-seg-shuffle.rb +3 -2
  29. data/lib/fairy/master/c-seg-split.rb +1 -1
  30. data/lib/fairy/master/c-seg-zip.rb +3 -1
  31. data/lib/fairy/master/c-sort.rb +7 -2
  32. data/lib/fairy/master/c-wc.rb +5 -3
  33. data/lib/fairy/node.rb +13 -2
  34. data/lib/fairy/node/p-barrier.rb +1 -1
  35. data/lib/fairy/node/p-basic-group-by.rb +22 -12
  36. data/lib/fairy/node/p-direct-product.rb +4 -2
  37. data/lib/fairy/node/p-filter.rb +8 -7
  38. data/lib/fairy/node/p-find.rb +2 -1
  39. data/lib/fairy/node/p-group-by.rb +17 -6
  40. data/lib/fairy/node/p-inject.rb +3 -2
  41. data/lib/fairy/node/p-output-file.rb +1 -1
  42. data/lib/fairy/node/p-seg-join.rb +2 -1
  43. data/lib/fairy/node/p-seg-zip.rb +2 -1
  44. data/lib/fairy/node/p-single-exportable.rb +3 -1
  45. data/lib/fairy/node/p-sort.rb +4 -2
  46. data/lib/fairy/node/p-task.rb +1 -1
  47. data/lib/fairy/node/p-wc.rb +5 -2
  48. data/lib/fairy/processor.rb +25 -18
  49. data/lib/fairy/share/block-source.rb +12 -2
  50. data/lib/fairy/share/conf.rb +35 -5
  51. data/lib/fairy/share/hash-simple-hash.rb +1 -1
  52. data/lib/fairy/share/log.rb +11 -4
  53. data/lib/fairy/share/pool-dictionary.rb +2 -1
  54. data/lib/fairy/share/port-marshaled-queue.rb +8 -1
  55. data/lib/fairy/share/port.rb +55 -45
  56. data/lib/fairy/share/reference.rb +2 -1
  57. data/lib/fairy/share/varray.rb +3 -1
  58. data/lib/fairy/share/vfile.rb +4 -2
  59. data/lib/fairy/version.rb +1 -1
  60. data/sample/sort.rb +69 -3
  61. data/spec/fairy8_spec.rb +1 -1
  62. data/test/testc.rb +380 -2
  63. data/tools/cap_recipe/Capfile +3 -3
  64. data/tools/fairy_conf_wizard.rb +375 -0
  65. data/tools/fairy_perf_graph.rb +15 -3
  66. data/tools/git-tag +1 -0
  67. data/tools/log-analysis.rb +59 -11
  68. metadata +33 -34
  69. data/ext/simple_hash/extconf.rb +0 -4
  70. data/ext/simple_hash/simple_hash.c +0 -42
@@ -61,4 +61,17 @@ module Fairy
61
61
  # end
62
62
 
63
63
  end
64
+
65
+ class CXGroupBy<CGroupBy
66
+ Controller.def_export self
67
+ def node_class_name
68
+ "PXGroupBy"
69
+ end
70
+ class CPostFilter<CGroupBy::CPostFilter
71
+ Controller.def_export self
72
+ def node_class_name
73
+ "PXGroupBy::PPostFilter"
74
+ end
75
+ end
76
+ end
64
77
  end
@@ -2,6 +2,7 @@
2
2
  #
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
+ require "xthread"
5
6
 
6
7
  require "fairy/master/c-io-filter"
7
8
  require "fairy/master/c-inputtable"
@@ -37,11 +38,11 @@ module Fairy
37
38
  end
38
39
  end
39
40
 
40
- class PortQueue<DelegateClass(Queue)
41
+ class PortQueue<DelegateClass(XThread::Queue)
41
42
  include Enumerable
42
43
 
43
44
  def initialize
44
- super(Queue.new)
45
+ super(XThread::Queue.new)
45
46
  end
46
47
 
47
48
  def each
@@ -3,6 +3,8 @@
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
5
 
6
+ require "xthread"
7
+
6
8
  require "fairy/master/c-io-filter"
7
9
  require "fairy/master/c-inputtable"
8
10
 
@@ -20,7 +22,7 @@ module Fairy
20
22
  @exports = {}
21
23
  @others_status = {}
22
24
  @exports_mutex = Mutex.new
23
- @exports_cv = ConditionVariable.new
25
+ @exports_cv = XThread::ConditionVariable.new
24
26
 
25
27
  @key_proc
26
28
  init_key_proc
@@ -1,6 +1,7 @@
1
1
  # encoding: UTF-8
2
2
 
3
3
  require "delegate"
4
+ require "xthread"
4
5
 
5
6
  require "fairy/master/c-filter"
6
7
  require "fairy/master/c-io-filter"
@@ -70,11 +71,11 @@ module Fairy
70
71
  end
71
72
  end
72
73
 
73
- class PortQueue<DelegateClass(Queue)
74
+ class PortQueue<DelegateClass(XThread::Queue)
74
75
  include Enumerable
75
76
 
76
77
  def initialize
77
- super(Queue.new)
78
+ super(XThread::Queue.new)
78
79
  end
79
80
 
80
81
  def each
@@ -21,7 +21,7 @@ module Fairy
21
21
 
22
22
  # @exports = []
23
23
  # @exports_mutex = Mutex.new
24
- # @exports_cv = ConditionVariable.new
24
+ # @exports_cv = XThread::ConditionVariable.new
25
25
 
26
26
  # @export2njob = {}
27
27
 
@@ -3,6 +3,8 @@
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
5
 
6
+ require "xthread"
7
+
6
8
  require "fairy/master/c-io-filter"
7
9
  require "fairy/master/c-inputtable"
8
10
 
@@ -23,7 +25,7 @@ module Fairy
23
25
  @exports = []
24
26
  @others_status = {}
25
27
  @exports_mutex = Mutex.new
26
- @exports_cv = ConditionVariable.new
28
+ @exports_cv = XThread::ConditionVariable.new
27
29
  end
28
30
 
29
31
  def opt_zip_by_substream?
@@ -2,6 +2,7 @@
2
2
  #
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
+ require "xthread"
5
6
 
6
7
  require "fairy/master/c-io-filter"
7
8
  require "fairy/master/c-inputtable"
@@ -20,7 +21,7 @@ module Fairy
20
21
 
21
22
  @pvs = nil
22
23
  @pvs_mutex = Mutex.new
23
- @pvs_cv = ConditionVariable.new
24
+ @pvs_cv = XThread::ConditionVariable.new
24
25
  end
25
26
 
26
27
  def get_pvs(buf=nil)
@@ -59,12 +60,16 @@ module Fairy
59
60
  key_proc = BBlock.new(@block_source, @context, self)
60
61
  end
61
62
 
62
- sorted = @samplings.flatten(1).sort_by{|e| key_proc.call(e)}
63
+ sorted = @samplings.flatten(1).map{|e| key_proc.call(e)}.sort_by{|e| e}
63
64
 
64
65
  #Log::debugf(self, "%s", sorted.inspect)
65
66
  idxes = (1...no_segment).collect{|i| (sorted.size*i).div(no_segment)}
66
67
  @pvs_mutex.synchronize do
67
68
  @pvs = sorted.values_at(*idxes)
69
+ sorted.clear
70
+ sorted = nil
71
+ @samplings.clear
72
+ @samplings = nil
68
73
  @pvs_cv.broadcast
69
74
  end
70
75
  end
@@ -3,6 +3,8 @@
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
5
 
6
+ require "xthread"
7
+
6
8
  require "fairy/master/c-io-filter"
7
9
  require "fairy/master/c-input"
8
10
  require "fairy/share/vfile"
@@ -27,10 +29,10 @@ module Fairy
27
29
  # key -> [export, ...]
28
30
  @exports = {}
29
31
  @exports_mutex = Mutex.new
30
- @exports_cv = ConditionVariable.new
32
+ @exports_cv = XThread::ConditionVariable.new
31
33
 
32
- # @pre_exports_queue = Queue.new
33
- @exports_queue = Queue.new
34
+ # @pre_exports_queue = XThread::Queue.new
35
+ @exports_queue = XThread::Queue.new
34
36
 
35
37
  @each_export_by_thread = nil
36
38
  @each_export_by_thread_mutex = Mutex.new
data/lib/fairy/node.rb CHANGED
@@ -7,11 +7,15 @@
7
7
 
8
8
  require "timeout"
9
9
 
10
+ require "xthread"
11
+ require "fiber-mon"
12
+
10
13
  require "deep-connect"
11
14
 
12
15
  require "fairy/version"
13
16
  require "fairy/share/conf"
14
17
 
18
+
15
19
  #DeepConnect::Organizer.immutable_classes.push Array
16
20
 
17
21
  module Fairy
@@ -27,11 +31,11 @@ module Fairy
27
31
 
28
32
  @processors = []
29
33
  @processors_mutex = Mutex.new
30
- @processors_cv = ConditionVariable.new
34
+ @processors_cv = XThread::ConditionVariable.new
31
35
 
32
36
  @active_processors = {}
33
37
  @active_processors_mutex = Mutex.new
34
- @active_processors_cv = ConditionVariable.new
38
+ @active_processors_cv = XThread::ConditionVariable.new
35
39
  end
36
40
 
37
41
  attr_accessor :id
@@ -67,6 +71,13 @@ module Fairy
67
71
  Log::info(self, "\tfairy version: #{Version}")
68
72
  Log::info(self, "\t[Powered BY #{RUBY_DESCRIPTION}]")
69
73
 
74
+ begin
75
+ require "fairy.so"
76
+ Log::warn self, "\t Load fairy.so"
77
+ rescue LoadError
78
+ Log::warn self, "Can't load fairy.so. Can't use this feature"
79
+ end
80
+
70
81
  @master.register_node(self)
71
82
  end
72
83
 
@@ -22,7 +22,7 @@ module Fairy
22
22
 
23
23
  # def input=(input)
24
24
  # unless @import
25
- # @import = Import.new(Queue.new)
25
+ # @import = Import.new(XThread::Queue.new)
26
26
  # @import.no=input.no
27
27
  # @import.add_key(input.key)
28
28
  # @import.set_log_callback do |n|
@@ -2,6 +2,7 @@
2
2
  #
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
+ require "xthread"
5
6
 
6
7
  require "fairy/node/p-io-filter"
7
8
 
@@ -18,13 +19,25 @@ module Fairy
18
19
  @block_source = block_source
19
20
 
20
21
  @exports = {}
21
- @exports_queue = Queue.new
22
+ @exports_queue = XThread::Queue.new
22
23
 
23
24
  @counter = {}
24
25
 
25
26
  #start_watch_exports
26
27
  end
27
28
 
29
+ def init_key_proc
30
+ hash_opt = @opts[:grouping_optimize]
31
+ hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
32
+
33
+ if hash_opt
34
+ @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
35
+ else
36
+ @key_proc = BBlock.new(@block_source, @context, self)
37
+ end
38
+ @key_proc
39
+ end
40
+
28
41
  def add_export(key, export)
29
42
  @exports[key] = export
30
43
  # @exports_queue.push [key, export]
@@ -36,14 +49,7 @@ module Fairy
36
49
  Log::debug(self, "START_EXPORT")
37
50
 
38
51
  start do
39
- hash_opt = @opts[:grouping_optimize]
40
- hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
41
-
42
- if hash_opt
43
- @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
44
- else
45
- @key_proc = BBlock.new(@block_source, @context, self)
46
- end
52
+ init_key_proc
47
53
 
48
54
  policy = @opts[:postqueuing_policy]
49
55
  begin
@@ -98,7 +104,9 @@ module Fairy
98
104
  Log::debug(self, "G3")
99
105
  @exports.each_pair do |key, export|
100
106
  Log::debug(self, "G3.WAIT #{key}")
101
- export.fib_wait_finish(@wait_cv)
107
+ @terminate_mon.synchronize do
108
+ export.fib_wait_finish(@wait_cv)
109
+ end
102
110
  end
103
111
  Log::debug(self, "G4")
104
112
  self.status = ST_EXPORT_FINISH
@@ -121,7 +129,7 @@ module Fairy
121
129
  @key_proc = BBlock.new(@block_source, @context, self)
122
130
 
123
131
  @exports = {}
124
- @exports_queue = Queue.new
132
+ @exports_queue = XThread::Queue.new
125
133
 
126
134
  # start_watch_exports
127
135
  end
@@ -227,7 +235,9 @@ Log::debug(self, "G3")
227
235
  @exports.each_pair do |key, export|
228
236
  next unless export
229
237
  Log::debug(self, "G4.WAIT #{key}")
230
- export.fib_wait_finish(@wait_cv)
238
+ @terminate_mon.synchronize do
239
+ export.fib_wait_finish(@wait_cv)
240
+ end
231
241
  end
232
242
  Log::debug(self, "G5")
233
243
  self.status = ST_EXPORT_FINISH
@@ -3,6 +3,8 @@
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
5
 
6
+ require "xthread"
7
+
6
8
  require "fairy/node/p-io-filter"
7
9
  require "fairy/node/p-single-exportable"
8
10
 
@@ -17,7 +19,7 @@ module Fairy
17
19
 
18
20
  @exports = nil
19
21
  @exports_mutex = Mutex.new
20
- @exports_cv = ConditionVariable.new
22
+ @exports_cv = XThread::ConditionVariable.new
21
23
  end
22
24
 
23
25
  def input=(input)
@@ -86,7 +88,7 @@ module Fairy
86
88
 
87
89
  @other_imports = nil
88
90
  @other_imports_mutex = Mutex.new
89
- @other_imports_cv = ConditionVariable.new
91
+ @other_imports_cv = XThread::ConditionVariable.new
90
92
  end
91
93
 
92
94
  def input=(input)
@@ -4,6 +4,7 @@
4
4
  #
5
5
 
6
6
  require "thread"
7
+ require "xthread"
7
8
 
8
9
  require "fairy/node/p-task"
9
10
 
@@ -59,18 +60,18 @@ module Fairy
59
60
 
60
61
  @no = nil
61
62
  @no_mutex = Mutex.new
62
- @no_cv = ConditionVariable.new
63
+ @no_cv = XThread::ConditionVariable.new
63
64
 
64
65
  @key = nil
65
66
  @key_mutex = Mutex.new
66
- @key_cv = ConditionVariable.new
67
+ @key_cv = XThread::ConditionVariable.new
67
68
 
68
69
  @status = ST_INIT
69
- @status_mon = processor.njob_mon
70
+ @status_mon = processor.njob_mon.new_mon
70
71
  @status_cv = @status_mon.new_cv
71
72
  notice_status(@status)
72
73
 
73
- @terminate_mon = processor.njob_mon
74
+ @terminate_mon = processor.njob_mon.new_mon
74
75
 
75
76
  @in_each = nil
76
77
  @in_each_mutex = Mutex.new
@@ -151,7 +152,7 @@ module Fairy
151
152
  end
152
153
 
153
154
  @main_thread = nil
154
- @terminate_mon.entry terminate_proc
155
+ processor.njob_mon.entry terminate_proc
155
156
  Log::info self, "FINISH PROCESSING: #{self.class}"
156
157
  end
157
158
  rescue Exception
@@ -282,7 +283,7 @@ module Fairy
282
283
  # 初期状態通知
283
284
  notice_status(@status)
284
285
 
285
- @status_mon.entry do
286
+ processor.njob_mon.entry do
286
287
  @status_mon.synchronize do
287
288
  old_status = nil
288
289
  loop do
@@ -297,7 +298,7 @@ module Fairy
297
298
  end
298
299
 
299
300
  def notice_status(st)
300
- @status_mon.entry do
301
+ processor.njob_mon.entry do
301
302
  @bjob.update_status(self, st)
302
303
  @ntask.update_status(self, st)
303
304
  end
@@ -2,6 +2,7 @@
2
2
  #
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
+ require "xthread"
5
6
 
6
7
  require "fairy/node/p-io-filter"
7
8
  require "fairy/node/p-single-exportable"
@@ -65,7 +66,7 @@ module Fairy
65
66
 
66
67
  @value = :__FAIRY_NO_VALUE__
67
68
  @value_mutex = Mutex.new
68
- @value_cv = ConditionVariable.new
69
+ @value_cv = XThread::ConditionVariable.new
69
70
  end
70
71
 
71
72
  def input=(input)
@@ -2,6 +2,7 @@
2
2
  #
3
3
  # Copyright (C) 2007-2010 Rakuten, Inc.
4
4
  #
5
+ require "xthread"
5
6
 
6
7
  require "fairy/node/p-io-filter"
7
8
  require "fairy/node/p-basic-group-by"
@@ -60,6 +61,15 @@ module Fairy
60
61
  end
61
62
  end
62
63
 
64
+ def init_key_proc
65
+ if @hash_optimize
66
+ @hash_proc = eval("proc{#{@block_source.source}}")
67
+ else
68
+ @hash_proc = BBlock.new(@block_source, @context, self)
69
+ end
70
+ @hash_proc
71
+ end
72
+
63
73
  # def start
64
74
  # super do
65
75
  # @key_value_buffer =
@@ -97,11 +107,7 @@ module Fairy
97
107
  def basic_each(&block)
98
108
  @key_value_buffer =
99
109
  eval("#{@buffering_policy[:buffering_class]}").new(self, @buffering_policy)
100
- if @hash_optimize
101
- @hash_proc = eval("proc{#{@block_source.source}}")
102
- else
103
- @hash_proc = BBlock.new(@block_source, @context, self)
104
- end
110
+ init_key_proc
105
111
 
106
112
  @input.each do |e|
107
113
  @key_value_buffer.push(e)
@@ -589,7 +595,7 @@ module Fairy
589
595
 
590
596
  def finish_wait
591
597
  @mx = Mutex.new
592
- @cv = ConditionVariable.new
598
+ @cv = XThread::ConditionVariable.new
593
599
  @mx.synchronize do
594
600
  @cv.wait(@mx)
595
601
  end
@@ -1053,6 +1059,11 @@ module Fairy
1053
1059
 
1054
1060
  read_buffer
1055
1061
  @key = @njob.hash_key(@cache.first)
1062
+ #Log::debug(self, "READ_KEY #{@key}");
1063
+ #unless @key
1064
+ # Log::debug(self, "READ_BUFFER #{@cache}");
1065
+ #end
1066
+
1056
1067
  end
1057
1068
 
1058
1069
  def_delegator :@io, :open