fairy 0.6.0 → 0.6.5

data/ext/xthread.h

@@ -0,0 +1,65 @@
+/**********************************************************************
+
+  xthread.h -
+
+  Copyright (C) 2011 Keiju Ishitsuka
+  Copyright (C) 2011 Penta Advanced Laboratories, Inc.
+
+**********************************************************************/
+
+
+#define XTHREAD_VERSION "0.1.2"
+
+RUBY_EXTERN VALUE rb_mXThread;
+RUBY_EXTERN VALUE rb_cXThreadFifo;
+RUBY_EXTERN VALUE rb_cXThreadConditionVariable;
+RUBY_EXTERN VALUE rb_cXThreadQueue;
+RUBY_EXTERN VALUE rb_cXThreadSizedQueue;
+RUBY_EXTERN VALUE rb_cXThreadMonitor;
+RUBY_EXTERN VALUE rb_cXThreadMonitorCond;
+
+
+RUBY_EXTERN VALUE rb_xthread_fifo_new(void);
+RUBY_EXTERN VALUE rb_xthread_fifo_empty_p(VALUE);
+RUBY_EXTERN VALUE rb_xthread_fifo_push(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_xthread_fifo_pop(VALUE);
+RUBY_EXTERN VALUE rb_xthread_fifo_clear(VALUE);
+RUBY_EXTERN VALUE rb_xthread_fifo_length(VALUE);
+
+RUBY_EXTERN VALUE rb_xthread_cond_new(void);
+RUBY_EXTERN VALUE rb_xthread_cond_signal(VALUE);
+RUBY_EXTERN VALUE rb_xthread_cond_broadcast(VALUE);
+RUBY_EXTERN VALUE rb_xthread_cond_wait(VALUE, VALUE, VALUE);
+
+RUBY_EXTERN VALUE rb_xthread_queue_new(void);
+RUBY_EXTERN VALUE rb_xthread_queue_push(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_xthread_queue_pop(VALUE);
+RUBY_EXTERN VALUE rb_xthread_queue_pop_non_block(VALUE);
+RUBY_EXTERN VALUE rb_xthread_queue_empty_p(VALUE);
+RUBY_EXTERN VALUE rb_xthread_queue_clear(VALUE);
+RUBY_EXTERN VALUE rb_xthread_queue_length(VALUE);
+
+RUBY_EXTERN VALUE rb_xthread_sized_queue_new(long);
+RUBY_EXTERN VALUE rb_xthread_sized_queue_max(VALUE);
+RUBY_EXTERN VALUE rb_xthread_sized_queue_set_max(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_xthread_sized_queue_push(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_xthread_sized_queue_pop(VALUE);
+RUBY_EXTERN VALUE rb_xthread_sized_queue_pop_non_block(VALUE);
+
+RUBY_EXTERN VALUE rb_xthread_monitor_new(void);
+RUBY_EXTERN VALUE rb_xthread_monitor_try_enter(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_enter(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_exit(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_synchronize(VALUE, VALUE (*)(VALUE), VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_new_cond(VALUE);
+
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_new(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_wait(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_wait_while(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_wait_until(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_signal(VALUE);
+RUBY_EXTERN VALUE rb_xthread_monitor_cond_broadcast(VALUE self);
+
+
+
+

data/fairy.gemspec

@@ -27,15 +27,18 @@ Gem::Specification.new do |s|
   s.files.concat ["etc/fairy.conf.tmpl"]
   s.files.concat Dir.glob("bin/{#{s.executables.grep(/.*[a-z]$/).join(",")}}")
   s.files.concat Dir.glob("bin/subcmd/*[A-Za-z]")
-  s.files.concat Dir.glob("ext/**/{Makefile,*.rb,*.c}")
+  s.files.concat Dir.glob("ext/{*.rb,*.c,*.h}")
   s.files.concat Dir.glob("doc/*.{rd,html}")
   s.files.concat Dir.glob("spec/{README,*.rb,run_all.sh}")
   s.files.concat Dir.glob("sample/*.rb")
   s.files.concat ["test/testc.rb"]
   s.files.concat Dir.glob("tools/**/*[a-z]")
 
+  s.add_dependency("xthread", ">= 0.1.4.001")
+  s.add_dependency("fiber-mon", ">= 0.2.1")
   s.add_dependency("DeepConnect", ">= 0.4.06")
-  s.add_dependency("fiber-mon", ">= 0.1.0")
+
+  s.extensions = ["ext/extconf.rb"]
   
   s.description = <<EOF
 fairy is a framework for distributed processing in Ruby, originally

data/lib/fairy.rb

@@ -4,6 +4,8 @@
 #
 
 require "thread"
+require "xthread"
+require "fiber-mon"
 
 require "deep-connect"
 
@@ -88,6 +90,14 @@ module Fairy
       Log::info self, "\tfairy version: #{Version}"
       Log::info(self, "\t[Powered By #{RUBY_DESCRIPTION}]") 
 
+
+      begin
+	require "fairy.so"
+	Log::warn self, "\t Load fairy.so"
+      rescue LoadError
+	Log::warn self, "Can't load fairy.so. Can't use this feature"
+      end
+
       @stdout_mutex = Mutex.new
 
       if CONF.DEBUG_MONITOR_ON
@@ -203,4 +213,3 @@ require "fairy/client/filter"
 require "fairy/client/input"
 
 require "fairy/client/addins"
-

data/lib/fairy/client/group-by.rb

@@ -11,17 +11,30 @@ module Fairy
 
     module Interface
       def group_by(hash_block, opts = nil)
+	
+	klass_name = opts[:group_by] if opts
+	klass_name ||= CONF.GROUP_BY
+	klass = ::Fairy::const_get(klass_name)
+
 	hash_block = BlockSource.new(hash_block) 
-	mod_group_by = GroupBy.new(@fairy, opts, hash_block)
+	mod_group_by = klass.new(@fairy, opts, hash_block)
 	mod_group_by.input = self
 	mod_group_by
       end
+
+      def xgroup_by(hash_block, opts = nil)
+	hash_block = BlockSource.new(hash_block) 
+	xgroup_by = XGroupBy.new(@fairy, opts, hash_block)
+	xgroup_by.input = self
+	xgroup_by
+      end
     end
     Fairy::def_filter_interface Interface
     ::Fairy::def_post_initialize{post_initialize}
 
     UnhandleMethods = [
       :post_mod_group_by_filter,
+      :post_mod_xgroup_by_filter,
       :post_merge_group_by_filter
     ]
     def self.post_initialize
@@ -90,6 +103,49 @@ module Fairy
 #   end
   
   end
+
+  class XGroupBy<GroupBy
+    def backend_class_name
+      "CXGroupBy"
+    end
+
+    UnhandleMethods = [
+      :post_mod_group_by_filter,
+      :post_mod_xgroup_by_filter,
+      :post_merge_group_by_filter
+    ]
+    def self.post_initialize
+      for interface in ::Fairy::FilterInterfaces
+	for m in interface.instance_methods
+	  m = m.intern if m.kind_of?(String)
+	  next if UnhandleMethods.include?(m)
+	
+	  m = m.id2name
+	  XGroupBy::module_eval %{
+            def #{m}(*argv, &block)
+	      post_mod_xgroup_by_filter(@block_source, @opts).#{m}(*argv, &block)
+	    end
+          }
+	end
+      end
+    end
+    ::Fairy::def_post_initialize{post_initialize}
+
+    class PostFilter<GroupBy::PostFilter
+      module Interface
+	def post_mod_xgroup_by_filter(hash_block, opts = nil)
+	  post_mod_xgroup_by_filter = XGroupBy::PostFilter.new(@fairy, opts, hash_block)
+	  post_mod_xgroup_by_filter.input = self
+	  post_mod_xgroup_by_filter
+	end
+	Fairy::def_filter_interface Interface
+      end
+      
+      def backend_class_name
+	"CXGroupBy::CPostFilter"
+      end
+    end
+  end
 end
 
 
@@ -190,5 +246,4 @@ Fairy.def_filter(:mod_group_by3) do |fairy, input, block_source, opts = {}|
     },
 		   :postqueuing_policy => {:queuing_class => :OnMemoryQueue}
 )
-  
 end

data/lib/fairy/client/here.rb

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
+require "xthread"
 
 require "fairy/client/io-filter"
 require "fairy/share/port"
@@ -34,7 +35,7 @@ module Fairy
     def each(&block)
       policy = @opts[:prequeuing_policy]
       
-      imports = Queue.new
+      imports = XThread::Queue.new
 
       parent_thread = Thread.current
       

data/lib/fairy/controller.rb

@@ -6,6 +6,9 @@
 require "thread"
 require "forwardable"
 
+require "xthread"
+require "fiber-mon"
+
 require "deep-connect.rb"
 
 require "fairy/version"
@@ -55,20 +58,23 @@ module Fairy
 
       @create_processor_mutex = Mutex.new
 
+      # deepspace -> processor_id
+      @deepspace2processor_id = {}
+
       # processor -> no of reserve 
       @reserves = {}
       @reserves_mutex = Mutex.new
-      @reserves_cv = ConditionVariable.new
+      @reserves_cv = XThread::ConditionVariable.new
 
       # bjob -> [processor, ...]
       @bjob2processors = {}
       @bjob2processors_mutex = Mutex.new
-      @bjob2processors_cv = ConditionVariable.new
+      @bjob2processors_cv = XThread::ConditionVariable.new
 
       # processor -> no of active ntasks
       @no_active_ntasks = {}
       @no_active_ntasks_mutex = Mutex.new
-      @no_active_ntasks_cv = ConditionVariable.new
+      @no_active_ntasks_cv = XThread::ConditionVariable.new
 
       @pool_dict = PoolDictionary.new
     end
@@ -108,6 +114,13 @@ module Fairy
       Log::info(self, "\tfairy version: #{Version}")
       Log::info(self, "\t[Powered by #{RUBY_DESCRIPTION}") 
 
+      begin
+	require "fairy.so"
+	Log::warn self, "\t Load fairy.so"
+      rescue LoadError
+	Log::warn self, "Can't load fairy.so. Can't use this feature"
+      end
+
       @master.register_controller(self)
 
     end
@@ -182,6 +195,7 @@ Log::debug(self, "TERMINATE: #2.5.1")
 Log::debug(self, "TERMINATE: #2.5.1")
 	      begin
 Log::debug(self, "TERMINATE: #2.5.2")
+		@deepspace2processor_id[p.deepspace] += "(terminated)"
 		p.node.terminate_processor(p)
 Log::debug(self, "TERMINATE: #2.5.3")
 	      rescue
@@ -274,6 +288,11 @@ Log::debug(self, "TERMINATE: #5")
 	# クライアントがおなくなりになったら, こっちも死ぬよ
 	@master.terminate_controller(self)
       end
+
+      if @deepspace2processor_id[deepspace]
+	Log::info(self, "CONTROLLER: processor disconected(#{@deepspace2processor_id[deepspace]})")
+	@deepspace2processor_id.delete(deepspace)
+      end
     end
 
     # 
@@ -340,7 +359,9 @@ Log::debug(self, "Processor[#{processor.id}] => #{no_active_ntasks}")
     def create_processor(node, bjob, &block)
       @create_processor_mutex.synchronize do
 	processor = node.create_processor
-	processor.connect_controller(self, CONF)
+	proc_id = processor.connect_controller(self, CONF)
+	@deepspace2processor_id[processor.deepspace] = proc_id
+
 	@reserves_mutex.synchronize do
 	  @reserves[processor] = 1
 	end

data/lib/fairy/master.rb

@@ -7,6 +7,10 @@ require "thread"
 require "resolv"
 require "ipaddr"
 
+require "xthread"
+require "fiber-mon"
+
+
 require "deep-connect"
 #DeepConnect::Organizer.immutable_classes.push Array
 
@@ -21,14 +25,14 @@ module Fairy
 
 #       @clients = {}
 #       @clients_mutex = Mutex.new
-#       @clients_cv = ConditionVariable.new
+#       @clients_cv = XThread::ConditionVariable.new
 
       @controller_seq = -1
       @controller_seq_mutex = Mutex.new
 
       @controllers = {}
       @controllers_mutex = Mutex.new
-      @controllers_cv = ConditionVariable.new
+      @controllers_cv = XThread::ConditionVariable.new
 
 #      @clientds2controller = {}
 
@@ -41,7 +45,7 @@ module Fairy
 
       @no_of_active_processors = {}
       @no_of_active_processors_mutex = Mutex.new
-      @no_of_active_processors_cv = ConditionVariable.new
+      @no_of_active_processors_cv = XThread::ConditionVariable.new
 
     end
 
@@ -74,6 +78,13 @@ module Fairy
       Log.info(self, "Master Service Start")
       Log::info(self, "\tfairy version: #{Version}")
       Log::info(self, "\t[Powerd By #{RUBY_DESCRIPTION}]") 
+
+      begin
+	require "fairy.so"
+	Log::warn self, "\t Load fairy.so"
+      rescue LoadError
+	Log::warn self, "Can't load fairy.so. Can't use this feature"
+      end
     end
 
     def when_disconnected(deepspace, opts)
@@ -327,3 +338,6 @@ Log::debug(self, "LAISURED NODE E:")
     end
   end
 end
+
+
+

data/lib/fairy/master/c-basic-group-by.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 require "fairy/master/c-io-filter"
 require "fairy/master/c-inputtable"
 
@@ -19,10 +21,10 @@ module Fairy
       # key -> [export, ...]
       @exports = {}
       @exports_mutex = Mutex.new
-      @exports_cv = ConditionVariable.new
+      @exports_cv = XThread::ConditionVariable.new
 
 #      @pre_exports_queue = Queue.new
-      @exports_queue = Queue.new
+      @exports_queue = XThread::Queue.new
 
       @each_export_by_thread = nil
       @each_export_by_thread_mutex = Mutex.new

data/lib/fairy/master/c-cat.rb

@@ -6,6 +6,7 @@
 require "delegate"
 
 require "thread"
+require "xthread"
 
 require "fairy/master/c-filter"
 require "fairy/master/c-io-filter"
@@ -22,7 +23,7 @@ module Fairy
       @others = others
       @export_node_pairs_queues = nil
       @export_node_pairs_queues_mutex = Mutex.new
-      @export_node_pairs_queues_cv = ConditionVariable.new
+      @export_node_pairs_queues_cv = XThread::ConditionVariable.new
 
       @main_precat = CPreCat.new(controller, opts)
 
@@ -115,7 +116,7 @@ module Fairy
 
 #     def start_get_exports
 #       @export_node_pairs_queues = [@input, *@others].collect{|input|
-# 	export_node_pairs = Queue.new
+# 	export_node_pairs = XThread::Queue.new
 # 	Thread.start do
 # 	  input.each_export do |*export_node_pair|
 # 	    export_node_pairs.push export_node_pair

data/lib/fairy/master/c-direct-product.rb

@@ -5,6 +5,8 @@
 
 require "forwardable"
 
+require "xthread"
+
 require "deep-connect"
 
 require "fairy/master/c-io-filter"
@@ -35,7 +37,7 @@ module Fairy
 
       @prefilter_no_nodes = {}
       @prefilter_no_nodes_mutex = Mutex.new
-      @prefilter_no_nodes_cv = ConditionVariable.new
+      @prefilter_no_nodes_cv = XThread::ConditionVariable.new
     end
 
     attr_reader :other_prefilters
@@ -94,11 +96,11 @@ module Fairy
 	@no = 0
 	@exports = {}
 	@exports_mutex = Mutex.new
-#	@exports_cv = ConditionVariable.new
+#	@exports_cv = XThread::ConditionVariable.new
 
 	@products = nil
 	@products_mutex = Mutex.new
-	@products_cv = ConditionVariable.new
+	@products_cv = XThread::ConditionVariable.new
       end
 
       def main=(main)

data/lib/fairy/master/c-filter.rb

@@ -3,6 +3,8 @@
 # Copyright (C) 2007-2010 Rakuten, Inc.
 #
 
+require "xthread"
+
 module Fairy
   class CFilter
     Controller.def_export self
@@ -29,16 +31,16 @@ module Fairy
 
       @number_of_nodes = nil
 #      @number_of_nodes_mutex = Mutex.new
-#      @number_of_nodes_cv = ConditionVariable.new
+#      @number_of_nodes_cv = XThread::ConditionVariable.new
 
       @nodes = []
       @nodes_for_next_filters = []
       @nodes_mutex = Mutex.new
-      @nodes_cv = ConditionVariable.new
+      @nodes_cv = XThread::ConditionVariable.new
 
       @nodes_status = {}
       @nodes_status_mutex = Mutex.new
-      @nodes_status_cv = ConditionVariable.new
+      @nodes_status_cv = XThread::ConditionVariable.new
 
       @controller.register_bjob(self)