fairy 0.6.0 → 0.6.5

data/lib/fairy/share/pool-dictionary.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 module Fairy
   class PoolDictionary
@@ -9,7 +10,7 @@ module Fairy
     def initialize
       @pool = {}
       @pool_mutex = Mutex.new
-      @pool_cv = ConditionVariable.new
+      @pool_cv = XThread::ConditionVariable.new
     end
 
     attr_reader :pool_mutex

data/lib/fairy/share/port-marshaled-queue.rb

@@ -2,12 +2,13 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 module Fairy
 
   class MarshaledQueue
 
-    def initialize(policy, queues_mon = Monitor.new, queues_cv = queues_mon.new_cond)
+    def initialize(policy, queues_mon = XThread::Monitor.new, queues_cv = queues_mon.new_cond)
       @policy = policy
 
       @chunk_size = CONF.MARSHAL_QUEUE_CHUNK_SIZE
@@ -319,7 +320,10 @@ module Fairy
 
     def store_2ndmemory(ary)
       open_2ndmemory do |io|
+# log解析で使いたいときはデコメントする
+#	Log::debug(self, "START M.STORE")
 	Marshal.dump(ary, io)
+#	Log::debug(self, "FINISH M.STORE")
       end
     end
 
@@ -330,9 +334,12 @@ module Fairy
     end
 
     def restore_2ndmemory(buf)
+# log解析で使いたいときはデコメントする
+#      Log::debug(self, "START M.RESTORE")
       io = buf.open
       queue = Marshal.load(io)
       buf.close!
+#      Log::debug(self, "FINISH M.RESTORE")
       queue
     end
 

data/lib/fairy/share/port.rb

@@ -5,6 +5,8 @@
 
 require "forwardable"
 
+require "xthread"
+
 require "fiber-mon"
 require "fairy/share/fast-tempfile.rb"
 
@@ -80,6 +82,7 @@ module Fairy
       end
 
       Log::debug(self, "Using Buffer: #{@queue.class}")
+Log::debug(self, "Inspect: #{@queue.inspect}")
       
       @log_import_ntimes_pop = CONF.LOG_IMPORT_NTIMES_POP
       @log_callback_proc = nil
@@ -88,7 +91,7 @@ module Fairy
 
       @no = nil
       @no_mutex = Mutex.new
-      @no_cv = ConditionVariable.new
+      @no_cv = XThread::ConditionVariable.new
 
       @key = nil
 
@@ -321,39 +324,44 @@ module Fairy
     END_OF_STREAM = :END_OF_STREAM
 
     ExportMonitor = FiberMon.new
-    ExportMonitor.start
+    #ExportMonitor.start
 
     def initialize(policy = nil)
-      
+
       @queuing_policy = policy
       @queuing_policy ||= CONF.POSTQUEUING_POLICY
       @max_chunk = CONF.POSTQUEUE_MAX_TRANSFER_SIZE
 
       @STR_TRANSFAR = CONF.TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE
+      if @queuing_policy.kind_of?(Hash) && 
+	  @queuing_policy[:transfar_marshal_string_array_optimize]
+	@STR_TRANSFAR = @queuing_policy[:transfar_marshal_string_array_optimize]
+      end
 
 #      @output_buf = []
 #      @output_buf_mutex = Mutex.new
-#      @output_buf_cv = ConditionVariable.new
+#      @output_buf_cv = XThread::ConditionVariable.new
       
       @output = nil
       @output_mutex = Mutex.new
-      @output_cv = ConditionVariable.new
+      @output_cv = XThread::ConditionVariable.new
 
       @njob_id = nil
 
       @no = nil
       @no_mutex = Mutex.new
-      @no_cv = ConditionVariable.new
+      @no_cv = XThread::ConditionVariable.new
 
       @key = nil
 
       @status = nil
       @status_mutex = Mutex.new
-      @status_cv = ConditionVariable.new
+      @status_cv = XThread::ConditionVariable.new
 
       @export_mon = ExportMonitor
-      @pop_cv = @export_mon.new_cv
-      @export_cv = @export_mon.new_cv
+      @export_mx = @export_mon.new_mon
+      @pop_cv = @export_mx.new_cv
+      @export_cv = @export_mx.new_cv
       
       case @queuing_policy
       when Hash
@@ -366,7 +374,6 @@ module Fairy
       end
 
       Log::debug(self, "Using Buffer: #{@queue.class}")
-
     end
 
     def log_id
@@ -464,11 +471,11 @@ module Fairy
     end
 
     def fib_pop
-      @export_mon.synchronize do
+      @export_mx.synchronize do
 	e = nil
 	#@export_mon.entry{e = @queue.pop; @pop_cv.signal}
 	Thread.start do
-	  @export_mon.synchronize do
+	  @export_mx.synchronize do
 	    e = @queue.pop
 	    @pop_cv.signal
 	  end
@@ -479,11 +486,11 @@ module Fairy
     end
 
     def fib_pop_all
-      @export_mon.synchronize do
+      @export_mx.synchronize do
 	e = nil
 	#@export_mon.entry{e = @queue.pop; @pop_cv.signal}
 	Thread.start do
-	  @export_mon.synchronize do
+	  @export_mx.synchronize do
 	    e = @queue.pop_all
 	    @pop_cv.signal
 	  end
@@ -506,6 +513,7 @@ module Fairy
 
     def start_export0
       @export_mon.entry do
+puts "AAAAAAAAAAAAAAAA:0"
 	if bug49 = CONF.DEBUG_BUG49
 	  # BUG#49用
 	  Log::debug(self, "export START")
@@ -514,7 +522,7 @@ module Fairy
 	end
 	self.status = :EXPORT
 
-	@export_mon.synchronize do
+	@export_mx.synchronize do
 #	  while (e = fib_pop) != END_OF_STREAM
 	  while (e = @queue.pop) != END_OF_STREAM
 	    if bug49
@@ -527,11 +535,11 @@ module Fairy
 	    begin 
 	      if PORT_KEEP_IDENTITY_CLASS_SET[e.class]
 		@output.asyncronus_send_with_callback(:push_keep_identity, e){
-		  @export_mon.synchronize{@export_cv.broadcast}
+		  @export_mx.synchronize{@export_cv.broadcast}
 		}
 	      else
 		@output.asyncronus_send_with_callback(:push, e) {
-		  @export_mon.synchronize{@export_cv.broadcast}
+		  @export_mx.synchronize{@export_cv.broadcast}
 		}
 	      end
 	      @export_cv.wait
@@ -567,15 +575,16 @@ module Fairy
     def start_export
       Log::debug(self, "START EXPORT")
 
-      unless @queue.respond_to?(:pop_all)
-	return start_export0
-      end
-
       if @queue.respond_to?(:pop_raw)
 	return start_export_raw
       end
+
+      unless @queue.respond_to?(:pop_all)
+	return start_export0
+      end
       
       @export_mon.entry do
+puts "AAAAAAAAAAAAAAAA:1"
 	if bug49 = CONF.DEBUG_BUG49
 	  # BUG#49用
 	  Log::debug(self, "export key=#{@key}: START")
@@ -583,7 +592,7 @@ module Fairy
 	  mod = CONF.LOG_IMPORT_NTIMES_POP
 	  limit = mod
 	end
-#	@export_mon.synchronize do
+#	@export_mx.synchronize do
 	  while (pops = @queue.pop_all).last != END_OF_STREAM
 #	  while (pops = fib_pop_all).last != END_OF_STREAM
 
@@ -683,9 +692,9 @@ module Fairy
 	if PORT_KEEP_IDENTITY_CLASS_SET[e.class]
 	  exports_elements_sub(elements, start, idx-1)
 	  sended = nil
-	  @export_mon.synchronize do
+	  @export_mx.synchronize do
 	    @output.asynchronus_send_with_callback(:push_keep_identity, e){
-	      @export_mon.synchronize do
+	      @export_mx.synchronize do
 		sended = true
 		@export_cv.broadcast
 	      end
@@ -707,10 +716,10 @@ module Fairy
     def exports_elements_sub(elements, start, last, max = @max_chunk)
       while last >= start
 	len = [max, last - start + 1].min
-	@export_mon.synchronize do
+	@export_mx.synchronize do
 	  sended = nil
 	  @output.asynchronus_send_with_callback(:push_buf, elements[start, len]){
-	    @export_mon.synchronize do
+	    @export_mx.synchronize do
 	      sended = true
 	      @export_cv.broadcast
 	    end
@@ -727,10 +736,10 @@ module Fairy
 	bigstr = elements[start, len].collect{|e| 
 	  e.gsub(/[\\\t]/){|v| v == "\t" ? "\\t" : '\\\\'}
 	}.join("\t")
-	@export_mon.synchronize do
+	@export_mx.synchronize do
 	  sended = nil
 	  @output.asynchronus_send_with_callback(:push_strings, bigstr) {
-	    @export_mon.synchronize do
+	    @export_mx.synchronize do
 	      sended = true
 	      @export_cv.broadcast
 	    end
@@ -740,12 +749,12 @@ module Fairy
 	start += len
       end
     end
-
-
+    
     def start_export_raw
       Log::debug(self, "START EXPORT(RAW MODE)")
 
       @export_mon.entry do
+puts "AAAAAAAAAAAAAAAA:2"
 	if bug49 = CONF.DEBUG_BUG49
 	  # BUG#49用
 	  Log::debug(self, "export key=#{@key}: START")
@@ -753,7 +762,7 @@ module Fairy
 	  mod = CONF.LOG_IMPORT_NTIMES_POP
 	  limit = mod
 	end
-#	@export_mon.synchronize do
+#	@export_mx.synchronize do
 	while (raw = @queue.pop_raw) != END_OF_STREAM
 	  if bug49
 	    n += pops.size
@@ -794,10 +803,10 @@ module Fairy
     end
 
     def export_elements_raw(raw)
-      @export_mon.synchronize do
+      @export_mx.synchronize do
 	sended = nil
 	@output.asynchronus_send_with_callback(:push_raw, raw) {
-	  @export_mon.synchronize do
+	  @export_mx.synchronize do
 	    sended = true
 	    @export_cv.broadcast
 	  end
@@ -862,7 +871,8 @@ module Fairy
 # ここから, 後で要検討
     def status=(val)
       @status_mutex.synchronize do
-	@status_cv.broadcast{@status = val}
+	@status = val
+	@status_cv.broadcast
       end
     end
 
@@ -888,12 +898,12 @@ module Fairy
       if queues_mon
 	raise "OnMemoryQueueはfiberをサポートしていません"
       end
-      Queue.new
+      XThread::Queue.new
     end
   end
 
   class PoolQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
       @policy = policy
 
       @queue_threshold = CONF.POOLQUEUE_POOL_THRESHOLD
@@ -951,7 +961,7 @@ module Fairy
 
       size = policy[:size]
       size ||= CONF.ONMEMORY_SIZEDQUEUE_SIZE
-      @queue = SizedQueue.new(size)
+      @queue = XThread::SizedQueue.new(size)
     end
 
     def_delegator :@queue, :push
@@ -960,7 +970,7 @@ module Fairy
   OnMemorySizedQueue = SizedQueue
 
   class SizedPoolQueue<PoolQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queues_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queues_mon.new_cond)
       super
       @max_size = policy[:size]
       @max_size ||= CONF.ONMEMORY_SIZEDQUEUE_SIZE
@@ -1007,7 +1017,7 @@ module Fairy
 
   class ChunkedPoolQueue
     # multi push threads single pop thread
-    def initialize(policy, queues_mon = Monitor.new, queues_cv = queues_mon.new_cond)
+    def initialize(policy, queues_mon = XThread::Monitor.new, queues_cv = queues_mon.new_cond)
       @policy = policy
 
       @queue_threshold = CONF.POOLQUEUE_POOL_THRESHOLD
@@ -1081,7 +1091,7 @@ module Fairy
   end
 
   class ChunkedSizedPoolQueue<ChunkedPoolQueue
-    def initialize(policy, queues_mon = Monitor.new, queues_cv = queues_mon.new_cond)
+    def initialize(policy, queues_mon = XThread::Monitor.new, queues_cv = queues_mon.new_cond)
       super
       @max_size = policy[:size]
       @max_size ||= CONF.ONMEMORY_SIZEDQUEUE_SIZE
@@ -1128,7 +1138,7 @@ module Fairy
   end
 
   class FileBufferdQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
 
       @policy = policy
       @threshold = policy[:threshold]
@@ -1249,7 +1259,7 @@ module Fairy
   end
 
   class ChunkedFileBufferdQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
       @policy = policy
       @threshold = policy[:threshold]
       @threshold ||= CONF.FILEBUFFEREDQUEUE_THRESHOLD
@@ -1379,7 +1389,7 @@ module Fairy
   end
 
   class SortedQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
       @policy = policy
 
       @pool_threshold = policy[:pool_threshold]
@@ -1540,7 +1550,7 @@ module Fairy
   end
 
   class OnMemorySortedQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
       @policy = policy
 
       @pool_threshold = policy[:pool_threshold]
@@ -1598,7 +1608,7 @@ module Fairy
   end
 
   class SortedQueue1<OnMemorySortedQueue
-    def initialize(policy, queue_mon = Monitor.new, queue_cv = queue_mon.new_cond)
+    def initialize(policy, queue_mon = XThread::Monitor.new, queue_cv = queue_mon.new_cond)
       super
 
       @threshold = policy[:threshold]

data/lib/fairy/share/reference.rb

@@ -4,6 +4,7 @@
 #
 
 require "thread"
+require "xthread"
 
 module Fairy
   class Reference
@@ -13,7 +14,7 @@ module Fairy
     def initialize
       @value = NULL_VALUE
       @value_mutex = Mutex.new
-      @value_cv = ConditionVariable.new
+      @value_cv = XThread::ConditionVariable.new
     end
 
     def value

data/lib/fairy/share/varray.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 module Fairy
   class VArray
     include Enumerable
@@ -16,7 +18,7 @@ module Fairy
       @arrays = []
       @arrays_size = arrays_size
       @arrays_mutex = Mutex.new
-      @arrays_cv = ConditionVariable.new
+      @arrays_cv = XThread::ConditionVariable.new
     end
 
     def size

data/lib/fairy/share/vfile.rb

@@ -5,6 +5,8 @@
 
 require "e2mmap"
 
+require "xthread"
+
 require "fairy/share/conf.rb"
 
 module Fairy
@@ -48,7 +50,7 @@ module Fairy
 
       @real_file_names = []
       @real_file_names_mutex = Mutex.new
-      @real_file_names_cv = ConditionVariable.new
+      @real_file_names_cv = XThread::ConditionVariable.new
 
       @base_name = nil
 
@@ -176,7 +178,7 @@ module Fairy
       @vfile_name = ary[0]
       @real_file_names = ary[1]
       @real_file_names_mutex = Mutex.new
-      @real_file_names_cv = ConditionVariable.new
+      @real_file_names_cv = XThread::ConditionVariable.new
     end
 
   end