fairy 0.6.0 → 0.6.5

data/lib/fairy/master/c-group-by.rb

@@ -61,4 +61,17 @@ module Fairy
 #   end
 
   end
+
+  class CXGroupBy<CGroupBy
+    Controller.def_export self
+    def node_class_name
+      "PXGroupBy"
+    end
+    class CPostFilter<CGroupBy::CPostFilter
+      Controller.def_export self
+      def node_class_name
+	"PXGroupBy::PPostFilter"
+      end
+    end
+  end
 end

data/lib/fairy/master/c-junction.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/master/c-io-filter"
 require "fairy/master/c-inputtable"
@@ -37,11 +38,11 @@ module Fairy
       end
     end
 
-    class PortQueue<DelegateClass(Queue)
+    class PortQueue<DelegateClass(XThread::Queue)
       include Enumerable
 
       def initialize
-	super(Queue.new)
+	super(XThread::Queue.new)
       end
 
       def each

data/lib/fairy/master/c-seg-join.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 require "fairy/master/c-io-filter"
 require "fairy/master/c-inputtable"
 
@@ -20,7 +22,7 @@ module Fairy
       @exports = {}
       @others_status = {}
       @exports_mutex = Mutex.new
-      @exports_cv = ConditionVariable.new
+      @exports_cv = XThread::ConditionVariable.new
 
       @key_proc
       init_key_proc

data/lib/fairy/master/c-seg-shuffle.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
 
 require "delegate"
+require "xthread"
 
 require "fairy/master/c-filter"
 require "fairy/master/c-io-filter"
@@ -70,11 +71,11 @@ module Fairy
       end
     end
 
-    class PortQueue<DelegateClass(Queue)
+    class PortQueue<DelegateClass(XThread::Queue)
       include Enumerable
 
       def initialize
-	super(Queue.new)
+	super(XThread::Queue.new)
       end
 
       def each

data/lib/fairy/master/c-seg-split.rb

@@ -21,7 +21,7 @@ module Fairy
 
 #      @exports = []
 #      @exports_mutex = Mutex.new
-#      @exports_cv = ConditionVariable.new
+#      @exports_cv = XThread::ConditionVariable.new
 
 #      @export2njob = {}
 

data/lib/fairy/master/c-seg-zip.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 require "fairy/master/c-io-filter"
 require "fairy/master/c-inputtable"
 
@@ -23,7 +25,7 @@ module Fairy
       @exports = []
       @others_status = {}
       @exports_mutex = Mutex.new
-      @exports_cv = ConditionVariable.new
+      @exports_cv = XThread::ConditionVariable.new
     end
 
     def opt_zip_by_substream?

data/lib/fairy/master/c-sort.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/master/c-io-filter"
 require "fairy/master/c-inputtable"
@@ -20,7 +21,7 @@ module Fairy
 
 	@pvs = nil
 	@pvs_mutex = Mutex.new
-	@pvs_cv = ConditionVariable.new
+	@pvs_cv = XThread::ConditionVariable.new
       end
       
       def get_pvs(buf=nil)
@@ -59,12 +60,16 @@ module Fairy
 	  key_proc = BBlock.new(@block_source, @context, self)
 	end
 
-	sorted = @samplings.flatten(1).sort_by{|e| key_proc.call(e)}
+	sorted = @samplings.flatten(1).map{|e| key_proc.call(e)}.sort_by{|e| e}
 
 #Log::debugf(self, "%s", sorted.inspect)
 	idxes = (1...no_segment).collect{|i| (sorted.size*i).div(no_segment)}
 	@pvs_mutex.synchronize do
 	  @pvs = sorted.values_at(*idxes)
+	  sorted.clear
+	  sorted = nil
+	  @samplings.clear
+	  @samplings = nil
 	  @pvs_cv.broadcast
 	end
       end

data/lib/fairy/master/c-wc.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 require "fairy/master/c-io-filter"
 require "fairy/master/c-input"
 require "fairy/share/vfile"
@@ -27,10 +29,10 @@ module Fairy
       # key -> [export, ...]
       @exports = {}
       @exports_mutex = Mutex.new
-      @exports_cv = ConditionVariable.new
+      @exports_cv = XThread::ConditionVariable.new
 
-#      @pre_exports_queue = Queue.new
-      @exports_queue = Queue.new
+#      @pre_exports_queue = XThread::Queue.new
+      @exports_queue = XThread::Queue.new
 
       @each_export_by_thread = nil
       @each_export_by_thread_mutex = Mutex.new

data/lib/fairy/node.rb

@@ -7,11 +7,15 @@
 
 require "timeout"
 
+require "xthread"
+require "fiber-mon"
+
 require "deep-connect"
 
 require "fairy/version"
 require "fairy/share/conf"
 
+
 #DeepConnect::Organizer.immutable_classes.push Array
 
 module Fairy
@@ -27,11 +31,11 @@ module Fairy
 
       @processors = []
       @processors_mutex = Mutex.new
-      @processors_cv = ConditionVariable.new
+      @processors_cv = XThread::ConditionVariable.new
 
       @active_processors = {}
       @active_processors_mutex = Mutex.new
-      @active_processors_cv = ConditionVariable.new
+      @active_processors_cv = XThread::ConditionVariable.new
     end
 
     attr_accessor :id
@@ -67,6 +71,13 @@ module Fairy
       Log::info(self, "\tfairy version: #{Version}")
       Log::info(self, "\t[Powered BY #{RUBY_DESCRIPTION}]") 
 
+      begin
+	require "fairy.so"
+	Log::warn self, "\t Load fairy.so"
+      rescue LoadError
+	Log::warn self, "Can't load fairy.so. Can't use this feature"
+      end
+
       @master.register_node(self)
     end
 

data/lib/fairy/node/p-barrier.rb

@@ -22,7 +22,7 @@ module Fairy
 
 #     def input=(input)
 #       unless @import
-# 	@import = Import.new(Queue.new)
+# 	@import = Import.new(XThread::Queue.new)
 # 	@import.no=input.no
 # 	@import.add_key(input.key)
 # 	@import.set_log_callback do |n| 

data/lib/fairy/node/p-basic-group-by.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 
@@ -18,13 +19,25 @@ module Fairy
       @block_source = block_source
 
       @exports = {}
-      @exports_queue = Queue.new
+      @exports_queue = XThread::Queue.new
       
       @counter = {}
 
       #start_watch_exports
     end
 
+    def init_key_proc
+      hash_opt = @opts[:grouping_optimize]
+      hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
+	
+      if hash_opt
+	@key_proc = eval("proc{#{@block_source.source}}", @context.binding)
+      else
+	@key_proc = BBlock.new(@block_source, @context, self)
+      end
+      @key_proc
+    end
+
     def add_export(key, export)
       @exports[key] = export
 #      @exports_queue.push [key, export]
@@ -36,14 +49,7 @@ module Fairy
       Log::debug(self, "START_EXPORT")
 
       start do
-	hash_opt = @opts[:grouping_optimize]
-	hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
-	
-	if hash_opt
-	  @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
-	else
-	  @key_proc = BBlock.new(@block_source, @context, self)
-	end
+	init_key_proc
 	
 	policy = @opts[:postqueuing_policy]
 	begin
@@ -98,7 +104,9 @@ module Fairy
       Log::debug(self, "G3")
       @exports.each_pair do |key, export|
 	Log::debug(self, "G3.WAIT #{key}")
-	export.fib_wait_finish(@wait_cv)
+	@terminate_mon.synchronize do
+	  export.fib_wait_finish(@wait_cv)
+	end
       end
       Log::debug(self, "G4")
       self.status = ST_EXPORT_FINISH
@@ -121,7 +129,7 @@ module Fairy
       @key_proc = BBlock.new(@block_source, @context, self)
 
       @exports = {}
-      @exports_queue = Queue.new
+      @exports_queue = XThread::Queue.new
 
 #      start_watch_exports
     end
@@ -227,7 +235,9 @@ Log::debug(self, "G3")
       @exports.each_pair do |key, export|
 	next unless export
 Log::debug(self, "G4.WAIT #{key}")
-	export.fib_wait_finish(@wait_cv)
+	@terminate_mon.synchronize do
+	  export.fib_wait_finish(@wait_cv)
+	end
       end
 Log::debug(self, "G5")
       self.status = ST_EXPORT_FINISH

data/lib/fairy/node/p-direct-product.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 require "fairy/node/p-io-filter"
 require "fairy/node/p-single-exportable"
 
@@ -17,7 +19,7 @@ module Fairy
 
 	@exports = nil
 	@exports_mutex = Mutex.new
-	@exports_cv = ConditionVariable.new
+	@exports_cv = XThread::ConditionVariable.new
       end
 
       def input=(input)
@@ -86,7 +88,7 @@ module Fairy
 	
 	@other_imports = nil
 	@other_imports_mutex = Mutex.new
-	@other_imports_cv = ConditionVariable.new
+	@other_imports_cv = XThread::ConditionVariable.new
       end
 
       def input=(input)

data/lib/fairy/node/p-filter.rb

@@ -4,6 +4,7 @@
 #
 
 require "thread"
+require "xthread"
 
 require "fairy/node/p-task"
 
@@ -59,18 +60,18 @@ module Fairy
 
       @no = nil
       @no_mutex = Mutex.new
-      @no_cv = ConditionVariable.new
+      @no_cv = XThread::ConditionVariable.new
 
       @key = nil
       @key_mutex = Mutex.new
-      @key_cv = ConditionVariable.new
+      @key_cv = XThread::ConditionVariable.new
 
       @status = ST_INIT
-      @status_mon = processor.njob_mon
+      @status_mon = processor.njob_mon.new_mon
       @status_cv = @status_mon.new_cv
       notice_status(@status)
 
-      @terminate_mon = processor.njob_mon
+      @terminate_mon = processor.njob_mon.new_mon
 
       @in_each = nil
       @in_each_mutex = Mutex.new
@@ -151,7 +152,7 @@ module Fairy
 	    end
 
 	    @main_thread = nil
-	    @terminate_mon.entry terminate_proc
+	    processor.njob_mon.entry terminate_proc
 	    Log::info self, "FINISH PROCESSING: #{self.class}"
 	  end
 	rescue Exception
@@ -282,7 +283,7 @@ module Fairy
       # 初期状態通知
       notice_status(@status)
 
-      @status_mon.entry do
+      processor.njob_mon.entry do
 	@status_mon.synchronize do
 	  old_status = nil
 	  loop do
@@ -297,7 +298,7 @@ module Fairy
     end
 
     def notice_status(st)
-      @status_mon.entry do
+      processor.njob_mon.entry do
 	@bjob.update_status(self, st)
 	@ntask.update_status(self, st)
       end

data/lib/fairy/node/p-find.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 require "fairy/node/p-single-exportable"
@@ -65,7 +66,7 @@ module Fairy
 
       @value = :__FAIRY_NO_VALUE__
       @value_mutex = Mutex.new
-      @value_cv = ConditionVariable.new
+      @value_cv = XThread::ConditionVariable.new
     end
 
     def input=(input)

data/lib/fairy/node/p-group-by.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 require "fairy/node/p-basic-group-by"
@@ -60,6 +61,15 @@ module Fairy
 	end
       end
 
+      def init_key_proc
+	if @hash_optimize
+	  @hash_proc = eval("proc{#{@block_source.source}}")
+	else
+	  @hash_proc = BBlock.new(@block_source, @context, self)
+	end
+	@hash_proc
+      end
+
 #       def start
 # 	super do
 # 	  @key_value_buffer = 
@@ -97,11 +107,7 @@ module Fairy
       def basic_each(&block)
 	@key_value_buffer = 
 	  eval("#{@buffering_policy[:buffering_class]}").new(self, @buffering_policy)
-	if @hash_optimize
-	  @hash_proc = eval("proc{#{@block_source.source}}")
-	else
-	  @hash_proc = BBlock.new(@block_source, @context, self)
-	end
+	init_key_proc
 
 	@input.each do |e|
 	  @key_value_buffer.push(e)
@@ -589,7 +595,7 @@ module Fairy
 
       def finish_wait
 	@mx = Mutex.new
-	@cv = ConditionVariable.new
+	@cv = XThread::ConditionVariable.new
 	@mx.synchronize do
 	  @cv.wait(@mx)
 	end
@@ -1053,6 +1059,11 @@ module Fairy
 
 	  read_buffer
 	  @key = @njob.hash_key(@cache.first)
+#Log::debug(self, "READ_KEY #{@key}");
+#unless @key
+#  Log::debug(self, "READ_BUFFER #{@cache}");
+#end
+
 	end
 
 	def_delegator :@io, :open