fairy 0.6.0 → 0.6.5

data/lib/fairy/node/p-inject.rb

@@ -2,8 +2,9 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
-require "fairy/node/p-filter"
+require "fairy/node/p-io-filter"
 require "fairy/node/p-single-exportable"
 
 module Fairy
@@ -72,7 +73,7 @@ module Fairy
 
       @value = :__FAIRY_NO_VALUE__
       @value_mutex = Mutex.new
-      @value_cv = ConditionVariable.new
+      @value_cv = XThread::ConditionVariable.new
     end
 
     def input=(input)

data/lib/fairy/node/p-output-file.rb

@@ -17,7 +17,7 @@ module Fairy
       super
       @vfile = vf
 
-#      @imports = Queue.new
+#      @imports = XThread::Queue.new
     end
 
     def input=(input)

data/lib/fairy/node/p-seg-join.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 require "fairy/node/p-single-exportable"
@@ -16,7 +17,7 @@ module Fairy
 
       @join_imports = nil
       @join_imports_mutex = Mutex.new
-      @join_imports_cv = ConditionVariable.new
+      @join_imports_cv = XThread::ConditionVariable.new
     end
 
     def join_imports

data/lib/fairy/node/p-seg-zip.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 require "fairy/node/p-single-exportable"
@@ -20,7 +21,7 @@ module Fairy
 
       @zip_imports = nil
       @zip_imports_mutex = Mutex.new
-      @zip_imports_cv = ConditionVariable.new
+      @zip_imports_cv = XThread::ConditionVariable.new
     end
 
     def zip_imports

data/lib/fairy/node/p-single-exportable.rb

@@ -75,7 +75,9 @@ module Fairy
 
     def wait_export_finish
       self.status = ST_WAIT_EXPORT_FINISH
-      @export.fib_wait_finish(@wait_cv)
+      @terminate_mon.synchronize do
+	@export.fib_wait_finish(@wait_cv)
+      end
       self.status = ST_EXPORT_FINISH
     end
   end

data/lib/fairy/node/p-sort.rb

@@ -2,9 +2,11 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 require "fairy/node/p-basic-group-by"
+require "fairy/node/p-single-exportable"
 
 module Fairy
   module PSort
@@ -25,7 +27,7 @@ module Fairy
 	    block.call(idx, item) if item
 	  end
 	end
-	@exports_queue = Queue.new
+	@exports_queue = XThread::Queue.new
 	
 	@counter = []
 
@@ -130,7 +132,7 @@ Log::debugf(self, "%s", @pvs.inspect)
 	  return
 	end
 	
-	unless idx = @pvs.find_index{|pv| key < @key_proc.call(pv)}
+	unless idx = @pvs.find_index{|pv| key < pv}
 	  idx = @pvs.size
 	end
 

data/lib/fairy/node/p-task.rb

@@ -24,7 +24,7 @@ module Fairy
       @njob_seq_mutex = Mutex.new
 
       @status = ST_INIT
-      @status_mon = processor.njob_mon
+      @status_mon = processor.njob_mon.new_mon
       @status_cv = @status_mon.new_cv
 
       start_watch_status

data/lib/fairy/node/p-wc.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/node/p-io-filter"
 
@@ -25,7 +26,7 @@ module Fairy
       @file = nil
 
       @exports = {}
-      @exports_queue = Queue.new
+      @exports_queue = XThread::Queue.new
       
       @counter = {}
 
@@ -145,7 +146,9 @@ Log::debug(self, "G3")
 Log::debug(self, "G4")
       for key,  export in @exports
 Log::debug(self, "G4.WAIT #{key}")
-	export.fib_wait_finish(@wait_cv)
+	@terminate_mon.synchronze do
+	  export.fib_wait_finish(@wait_cv)
+	end
       end
 Log::debug(self, "G5")
       self.status = ST_EXPORT_FINISH

data/lib/fairy/processor.rb

@@ -3,7 +3,9 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
 require "fiber-mon"
+
 require "deep-connect"
 
 require "fairy/version"
@@ -11,6 +13,7 @@ require "fairy/share/conf"
 require "fairy/share/stdout"
 
 
+
 #DeepConnect::Organizer.immutable_classes.push Array
 
 # require "node/nfile"
@@ -47,8 +50,8 @@ module Fairy
 	  ERR::Raise ERR::INTERNAL::CantDefExport, obj.to_s
 	end
       end
-
       EXPORTS.push [name, obj]
+
     end
 
     def initialize(id)
@@ -87,16 +90,14 @@ module Fairy
 
       @deepconnect = DeepConnect.start(service)
       @deepconnect.register_service("Processor", self)
+#      DeepConnect::Conf.DISPLAY_KEEP_ALIVE = true
 
       @deepconnect.when_disconnected do |deepspace, opts|
 	when_disconnected(deepspace, opts)
       end
 
-      for name, obj in EXPORTS
-	export(name, obj)
-      end
 
-      @njob_mon.start
+      #@njob_mon.start
 
       require "fairy/share/inspector"
       @deepconnect.export("Inspector", Inspector.new(self))
@@ -112,6 +113,17 @@ module Fairy
       Log::info(self, "\tfairy version: #{Version}")
       Log::info(self, "\t[Powered By #{RUBY_DESCRIPTION}]") 
 
+      begin
+	require "fairy.so"
+	Log::warn self, "\t Load fairy.so"
+      rescue LoadError
+	Log::warn self, "Can't load fairy.so. Can't use this feature"
+      end
+
+      for name, obj in EXPORTS
+	export(name, obj)
+      end
+
       start_watch_status
 
 
@@ -138,8 +150,10 @@ module Fairy
       end
 
       $stdout = Stdout.new(controller)
+
+      sprintf("%s %s#%d", Log::host, Log::type, Log::pid)
     end
-    DeepConnect.def_method_spec(self, "REF connect_controller(REF, DVAL)")
+    DeepConnect.def_method_spec(self, "DVAL connect_controller(REF, DVAL)")
 
     def terminate
       # clientが終了したときの終了処理
@@ -290,8 +304,8 @@ module Fairy
       @status = :ST_WAIT
       @ntask_status = {}
 
-#      @status_mutex = Mutex.new
-      @status_cv = @njob_mon.new_cv
+      @status_mx = @njob_mon.new_mon
+      @status_cv = @status_mx.new_cv
 
     end
 
@@ -354,41 +368,35 @@ module Fairy
 
 
     def update_status(ntask, st)
-Log::debug(self, "UPDATE_STATUS: #{ntask}, #{st}")
-      @njob_mon.synchronize do
+      Log::debug(self, "UPDATE_STATUS: #{ntask}, #{st}")
+      @status_mx.synchronize do
 	@ntask_status[ntask] = st
 
 	case st
 	when :ST_INIT
 	  # do nothing
 	  if all_ntasks_semiactivated?(:no_lock)
-Log::debug(self, "UPDATE_STATUS A: #{st}")
 	    @status = :ST_SEMIACTIVATE
 	  end
 	when :ST_WAIT_IMPORT
 	  if all_ntasks_semiactivated?(:no_lock)
-Log::debug(self, "UPDATE_STATUS B: #{st}")
 	    @status = :ST_SEMIACTIVATE
 	  end
 	when :ST_ACTIVATE
-Log::debug(self, "UPDATE_STATUS C: #{st}")
 	  @status = :ST_ACTIVATE
 	when :ST_ALL_IMPORTED, 
 	    :ST_WAIT_EXPORT_FINISH, 
 	    :ST_EXPORT_FINISH, 
 	    :ST_OUTPUT_FINISH
 	  if all_ntasks_semiactivated?(:no_lock)
-Log::debug(self, "UPDATE_STATUS D: #{st}")
 	    @status = :ST_SEMIACTIVATE
 	  end
 	when :ST_FINISH
 	  if all_ntasks_finished?(:no_lock)
-Log::debug(self, "UPDATE_STATUS E: #{st}")
 	    @status = :ST_WAIT
 	  end
 	else
 	  if @status == :ST_WAIT
-Log::debug(self, "UPDATE_STATUS F: #{st}")
 	    @status = :ST_ACTIVATE
 	  end
 	end
@@ -399,9 +407,8 @@ Log::debug(self, "UPDATE_STATUS F: #{st}")
     def start_watch_status
       # 初期状態通知
       notice_status(@status)
-
       @njob_mon.entry do
-	@njob_mon.synchronize do
+	@status_mx.synchronize do
 	  old_status = nil
 	  old_no_active_ntasks = 0
 	  loop do

data/lib/fairy/share/block-source.rb

@@ -108,7 +108,9 @@ module Fairy
 	else
 	  bt = $!.backtrace
 	  bt = bt.select{|l| /fairy.*(share|job|backend|node|processor|controller)|deep-connect|__FORWARDABLE__|bin.*processor/ !~ l} unless CONF.DEBUG_FULL_BACKTRACE
-	  bt.first.sub!("bind", @block_source.caller_method)
+	  unless bt.empty?
+	    bt.first.sub!("bind", @block_source.caller_method)
+	  end
 	  bt.push *@block_source.backtrace.dc_deep_copy
 	  $!.set_backtrace(bt)
 
@@ -125,7 +127,9 @@ module Fairy
 	end
 	bt = $!.backtrace
 	bt = bt.select{|l| /fairy.*(share|job|backend|node|processor|controller)|deep-connect|__FORWARDABLE__|bin.*processor/ !~ l} unless CONF.DEBUG_FULL_BACKTRACE
-	bt.first.sub!("bind", @block_source.caller_method)
+	unless bt.empty?
+	  bt.first.sub!("bind", @block_source.caller_method)
+	end
 	bt.push *@block_source.backtrace.dc_deep_copy
 	$!.set_backtrace(bt)
 
@@ -163,6 +167,9 @@ module Fairy
 	else
 	  bt = $!.backtrace
 	  bt = bt.select{|l| /fairy.*(share|job|backend|node|processor|controller)|deep-connect|__FORWARDABLE__|bin.*processor/ !~ l} unless CONF.DEBUG_FULL_BACKTRACE
+	  unless bt.empty?
+	    bt.first.sub!("bind", @block_source.caller_method)
+	  end
 	  bt.first.sub!("bind", @block_source.caller_method)
 	  bt.push *@block_source.backtrace.dc_deep_copy
 	  $!.set_backtrace(bt)
@@ -181,6 +188,9 @@ module Fairy
 	bt = $!.backtrace
 	bt = bt.select{|l| /fairy.*(share|job|backend|node|processor|controller)|deep-connect|__FORWARDABLE__|bin.*processor/ !~ l} unless CONF.DEBUG_FULL_BACKTRACE
 	bt.first.sub!("bind", @block_source.caller_method)
+	unless bt.empty?
+	  bt.first.sub!("bind", @block_source.caller_method)
+	end
 	bt.push *@block_source.backtrace.dc_deep_copy
 	$!.set_backtrace(bt)
 

data/lib/fairy/share/conf.rb

@@ -111,6 +111,16 @@ module Fairy
     def_prop :MARSHAL_QUEUE_MIN_CHUNK_NO
     def_prop :SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO
 
+    def_prop :XMARSHAL_QUEUE_CHUNK_SIZE
+    def_prop :XMARSHAL_QUEUE_MIN_CHUNK_NO
+    def_prop :XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT
+    def_prop :XMARSHAL_QUEUE_USE_STRING_BUFFER
+    def_prop :XMARSHAL_QUEUE_LOG_MSTORE
+
+    def_prop :XSIZED_QUEUE_CHUNK_SIZE
+    def_prop :XSIZED_QUEUE_QUEUES_LIMIT
+    def_prop :XSIZED_QUEUE_USE_STRING_BUFFER
+
     def_prop :SORTEDQUEUE_POOL_THRESHOLD
     def_prop :SORTEDQUEUE_THRESHOLD
     def_prop :SORTEDQUEUE_SORTBY
@@ -119,6 +129,7 @@ module Fairy
     def_prop :HERE_POOL_THRESHOLD
     #def_prop :OUTPUT_LOCAL_POOL_THRESHOLD
 
+    def_prop :GROUP_BY
     def_prop :GROUP_BY_NO_SEGMENT
     def_prop :GROUP_BY_HASH_MODULE
     def_prop :GROUP_BY_GROUPING_OPTIMIZE
@@ -127,6 +138,8 @@ module Fairy
     def_prop :GROUP_BY_CMSB_THRESHOLD
     def_prop :GROUP_BY_CMSB_CHUNK_SIZE
 
+    def_prop :XGROUP_BY_BUFFERING_POLICY
+
     def_prop :BARRIER_MEMORY_BUFFERING_POLICY
 
     def_prop :SORT_BUFFERING_POLICY
@@ -288,19 +301,32 @@ module Fairy
 
   DEFAULT_CONF.POSTMAPPING_POLICY = nil
 
-  DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
-  DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
+  DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :XSizedQueue}
+  DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :XMarshaledQueue}
 
   DEFAULT_CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
-  DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 10000
+  DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 100000
   DEFAULT_CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
   DEFAULT_CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
 
   DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
+
+  DEFAULT_CONF.XMARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE 
+  DEFAULT_CONF.XMARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO 
+  DEFAULT_CONF.XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT = 1000
+  DEFAULT_CONF.XMARSHAL_QUEUE_USE_STRING_BUFFER = true
+  DEFAULT_CONF.XMARSHAL_QUEUE_LOG_MSTORE = false
+
+  DEFAULT_CONF.XSIZED_QUEUE_CHUNK_SIZE = DEFAULT_CONF.XMARSHAL_QUEUE_CHUNK_SIZE
+  DEFAULT_CONF.XSIZED_QUEUE_QUEUES_LIMIT = 
+    DEFAULT_CONF.XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT
+  DEFAULT_CONF.XSIZED_QUEUE_USE_STRING_BUFFER = 
+    DEFAULT_CONF.XMARSHAL_QUEUE_USE_STRING_BUFFER
   
-  DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
+  DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = 
+    DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.SORTEDQUEUE_THRESHOLD = 10000/2
   DEFAULT_CONF.SORTEDQUEUE_SORTBY = %{|v| v}
 
@@ -308,6 +334,8 @@ module Fairy
   DEFAULT_CONF.HERE_POOL_THRESHOLD = 32000
   #DEFAULT_CONF.OUTPUT_LOCAL_POOL_THRESHOLD = 32000
 
+  DEFAULT_CONF.GROUP_BY = :XGroupBy
+
   DEFAULT_CONF.GROUP_BY_NO_SEGMENT = 4
   DEFAULT_CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
   DEFAULT_CONF.GROUP_BY_GROUPING_OPTIMIZE = false
@@ -316,9 +344,11 @@ module Fairy
   DEFAULT_CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
   DEFAULT_CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
 
+  DEFAULT_CONF.XGROUP_BY_BUFFERING_POLICY = {:buffering_class => :XDirectMergeSortBuffer}
+
   DEFAULT_CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
 
-  DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
+  DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::XDirectMergeSortBuffer"}
   DEFAULT_CONF.SORT_SAMPLING_MIN = 100
   DEFAULT_CONF.SORT_SAMPLING_MAX = 10000
   DEFAULT_CONF.SORT_SAMPLING_RATIO_1_TO = 100

data/lib/fairy/share/hash-simple-hash.rb

@@ -3,7 +3,7 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
-require "simple_hash"
+#require "simple_hash"
 
 module Fairy
   module HValueGenerator

data/lib/fairy/share/log.rb

@@ -4,6 +4,7 @@
 #
 
 require "thread"
+require "xthread"
 require "stringio"
 
 require "forwardable"
@@ -30,7 +31,7 @@ module Fairy
 
       @buffer = []
       @buffer_mutex = Mutex.new
-      @buffer_cv = ConditionVariable.new
+      @buffer_cv = XThread::ConditionVariable.new
 
       set_local_output_dev
       
@@ -163,9 +164,15 @@ module Fairy
 	sender = "UNDEF"
       end
       log(sender) do |sio|
-	sio.puts "#{exception.message}: #{exception.class}"
-	for l in exception.backtrace
-	  sio.puts l
+	if exception.kind_of?(Exception)
+	  sio.puts "#{exception.message}: #{exception.class}"
+	  for l in exception.backtrace
+	    sio.puts l
+	  end
+	else
+	  sio.puts "Unknown exception rised!!(Exp=#{exception.inspect})"
+	  sio.puts "Backtorace: "
+	  log_backtrace(sender)
 	end
       end
     end