fairy 0.6.0

data/lib/fairy/master/c-select.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CSelect<CIOFilter
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def node_class_name
+      "PSelect"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+
+  end
+end

data/lib/fairy/master/c-sort.rb

@@ -0,0 +1,108 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+require "fairy/master/c-basic-group-by"
+
+module Fairy
+
+  module CSort
+    class CPreSort<CBasicGroupBy
+      Controller.def_export self
+
+      def initialize(controller, opts, block_source)
+	super
+
+	@samplings = []
+
+	@pvs = nil
+	@pvs_mutex = Mutex.new
+	@pvs_cv = ConditionVariable.new
+      end
+      
+      def get_pvs(buf=nil)
+# BUG#271対応. 全てのセグメントからサンプルを取るのではなく, 最初のセ
+# グメントからのみサンプリングを取るようにした.
+	if buf
+	  @samplings.push buf
+
+# BUG#271対応. 
+#	if @samplings.size >= number_of_nodes
+#	    make_pvs
+#	end
+	  make_pvs
+	end
+
+	@pvs_mutex.synchronize do
+	  while !@pvs
+	    @pvs_cv.wait(@pvs_mutex)
+	  end
+	end
+	@pvs
+      end
+      DeepConnect.def_method_spec(self, "DVAL get_pvs(DVAL)")
+
+      def make_pvs
+	no_segment = @opts[:no_segment]
+	no_segment ||= Fairy::CONF.SORT_NO_SEGMENT
+
+
+	cmp_opt = @opts[:cmp_optimize]
+	cmp_opt = CONF.SORT_CMP_OPTIMIZE if cmp_opt.nil?
+	
+	if cmp_opt
+	  key_proc = eval("proc{#{@block_source.source}}", @context.binding)
+	else
+	  key_proc = BBlock.new(@block_source, @context, self)
+	end
+
+	sorted = @samplings.flatten(1).sort_by{|e| key_proc.call(e)}
+
+#Log::debugf(self, "%s", sorted.inspect)
+	idxes = (1...no_segment).collect{|i| (sorted.size*i).div(no_segment)}
+	@pvs_mutex.synchronize do
+	  @pvs = sorted.values_at(*idxes)
+	  @pvs_cv.broadcast
+	end
+      end
+
+      def node_class_name
+	"PSort::PPreSort"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+    end
+
+    class CPostSort<CIOFilter
+      Controller.def_export self
+
+      def initialize(controller, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def node_class_name
+	"PSort::PPostSort"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+
+      def create_import(processor)
+	policy = @opts[:postfilter_prequeuing_policy]
+	policy ||= @opts[:prequeuing_policy]
+	
+	processor.create_import(policy)
+      end
+
+    end
+
+  end
+end
+

data/lib/fairy/master/c-there.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-input"
+
+module Fairy
+  class CThere<CInput
+    Controller.def_export self
+
+    def initialize(controller, opts, enumerable)
+      super
+      @enumerable = enumerable
+    end
+
+    def node_class_name
+      "PThere"
+    end
+
+    def njob_creation_params
+#      [@enumerable]
+      []
+    end
+
+    def start
+      @cthere_place = CTherePlace.new(@enumerable)
+      start_create_nodes
+    end
+
+    def input
+      @cthere_place
+    end
+
+#     def create_and_start_nodes
+#       begin
+# 	no = 0
+# 	@create_node_mutex.synchronize do
+# 	  nthere = nil
+# 	  @controller.assign_new_processor(self) do |processor|
+# 	    nthere = create_node(processor)
+# 	  end
+# 	  no += 1
+# 	  nthere.start
+# 	end
+#       rescue BreakCreateNode
+# 	# do nothing
+# 	Log::debug self, "BREAK CREATE NODE: #{self}" 
+#       ensure
+# 	self.number_of_nodes = no
+#       end
+#     end
+  end
+end
+
+
+      

data/lib/fairy/master/c-wc.rb

@@ -0,0 +1,232 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-input"
+require "fairy/share/vfile"
+require "fairy/share/file-place"
+
+module Fairy
+  class CWC<CInput
+    Controller.def_export self
+
+    def CWC.open(controller, opts, descripter)
+      bfile = CWC.new(controller, opts)
+      bfile.open(desctipter)
+      bfile
+    end
+    DeepConnect.def_single_method_spec(self, "REF open(REF, VAL, VAL)")
+
+    def initialize(controller, opts)
+      super
+
+      @no_of_exports = 0
+
+      # key -> [export, ...]
+      @exports = {}
+      @exports_mutex = Mutex.new
+      @exports_cv = ConditionVariable.new
+
+#      @pre_exports_queue = Queue.new
+      @exports_queue = Queue.new
+
+      @each_export_by_thread = nil
+      @each_export_by_thread_mutex = Mutex.new
+
+      @hash_seed = controller.hash_seed
+    end
+
+    attr_reader :hash_seed
+
+    def node_class_name
+      "PWC"
+    end
+
+    def open(vf)
+      @vfile = vf
+      @cfile_place = CFilePlace.new(@vfile)
+
+      start_create_nodes
+    end
+    DeepConnect.def_method_spec(self, "REF open(DVAL)")
+
+    def input
+      @cfile_place
+    end
+
+    def start_create_nodes
+      super
+
+      start_watch_all_node_imported
+    end
+
+    def each_assigned_filter(&block)
+      super
+
+      @each_export_by_thread_mutex.synchronize do
+	@each_export_by_thread.join if @each_export_by_thread
+      end
+    end
+
+    def each_export_by(njob, mapper, &block)
+      @each_export_by_thread_mutex.synchronize do
+	return if @each_export_by_thread
+
+	@each_export_by_thread = Thread.start{
+	  # すべての njob がそろうまで待つ
+	  # 後段が先にスケジュールされてデッドロックするのを避けるため.
+	  number_of_nodes
+
+	  begin
+	    while pair = @exports_queue.pop
+	      exp, njob = pair
+Log::debug(self, "EXPORT_BY, #{exp.key}")
+	      block.call exp
+
+	      @exports_mutex.synchronize do
+		if @exports[exp.key].first == exp
+		  @exports[exp.key][1..-1].each do |e|
+		    e.output = exp.output
+		  end
+		end
+	      end
+	    end
+	  rescue
+	    Log::fatal_exception
+	    raise
+	  end
+	}
+      end
+    end
+
+    def bind_export(exp, imp)
+      # do nothing
+    end
+
+    def add_exports(key, export, njob)
+      @exports_mutex.synchronize do
+	if exports = @exports[key]
+	  export.output = exports.first.output if exports.first.output?
+	  export.no = exports.first.no
+	  exports.push export
+	else
+	  export.no = @no_of_exports
+	  @no_of_exports += 1
+	  @exports[key] = [export]
+	  @exports_queue.push [export, njob]
+#	  @pre_exports_queue.push [export, njob]
+	end
+      end
+    end
+
+    def update_exports(key, export, njob)
+      add_exports(key, export, njob)
+      nil
+    end
+
+    def start_watch_all_node_imported
+      Thread.start do
+	# すべての njob がそろうまで待つ
+	# 後段が先にスケジュールされてデッドロックするのを避けるため.
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: S")
+	number_of_nodes
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1")
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 2")
+	# すべての exports がそろうまで待つ
+	@nodes_status_mutex.synchronize do
+	  while !all_node_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+	@exports_queue.push nil
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 4")
+	for key, exports in @exports
+	  exports.first.output_no_import = exports.size
+	end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: E")
+      end
+      nil
+    end
+
+    def all_node_arrived?
+      @nodes_mutex.synchronize{@number_of_nodes}
+    end
+
+    def all_node_imported?
+      # すべてのnjobがそろったか?
+      return false unless @nodes_mutex.synchronize{@number_of_nodes}
+
+      each_node(:exist_only) do |node|
+	st = @nodes_status[node]
+	# こちらはNG: outputが設定されていないとまずい.
+	# すべてのnodeがそろったとしてもすべてのexportがそろっているとは限らない
+#	unless [:ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH, :ST_ALL_IMPORTED].include?(st)
+	unless [:ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH].include?(st)
+	  return false
+	end
+      end
+      true
+    end
+
+    class CPostFilter<CIOFilter
+      Controller.def_export self
+
+      def initialize(controller, opts)
+	super
+
+	@hash_seed = controller.hash_seed
+      end
+
+      attr_reader :hash_seed
+
+      def node_class_name
+	"PWC::PPostFilter"
+      end
+
+      def njob_creation_params
+	[@vfile]
+      end
+
+      def output(vf)
+	@vfile = vf
+      end
+
+      def njob_creation_params
+	[@vfile]
+      end
+      
+      def create_import(processor)
+	policy = @opts[:postfilter_prequeuing_policy]
+	policy ||= @opts[:prequeuing_policy]
+	
+	processor.create_import(policy)
+      end
+
+
+      def wait_all_output_finished
+	@nodes_status_mutex.synchronize do
+	  while !all_node_outputted?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+      end
+
+      def all_node_outputted?
+
+	return false unless @number_of_nodes
+
+	all_outputted = true
+	each_node(:exist_only) do |node|
+	  st = @nodes_status[node]
+	  all_outputted &&= [:ST_FINISH, :ST_OUTPUT_FINISH].include?(st)
+	end
+	all_outputted
+      end
+    end
+  end
+end

data/lib/fairy/master/job-interpriter.rb

@@ -0,0 +1,19 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  class JobInterpriter
+    def initialize(controller)
+      @controller = controller
+    end
+
+    def exec(atom)
+      puts "SEND: #{atom.receiver}.#{atom.message}(#{atom.args.map{|e| e.to_s}.join(",")})"
+      ret = atom.receiver.send(atom.message, *atom.args)
+      # このあと何か必要か?
+    end
+  end
+end
+