fairy 0.6.0

data/lib/fairy/master/c-map.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CMap<CIOFilter
+    Controller.def_export self
+
+    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, REF)")
+    def initialize(controller, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def node_class_name
+      "PMap"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+  end
+end

data/lib/fairy/master/c-merge-group-by.rb

@@ -0,0 +1,241 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+require "fairy/master/c-basic-group-by"
+
+require "fairy/share/port"
+
+module Fairy
+  class CMergeGroupBy<CBasicGroupBy
+    Controller.def_export self
+
+    def node_class_name
+      "PMergeGroupBy"
+    end
+
+    def each_export_by(njob, mapper, &block)
+      @each_export_by_thread_mutex.synchronize do
+	return if @each_export_by_thread
+
+	@each_export_by_thread = Thread.start{
+	  # すべての njob がそろうまで待つ
+	  # 後段が先にスケジュールされてデッドロックするのを避けるため.
+	  number_of_nodes
+
+	  begin
+	    while pair = @exports_queue.pop
+	      exp, njob = pair
+Log::debug(self, "EXPORT_BY, #{exp.key}")
+	      block.call exp
+	    end
+	  rescue
+	    Log::fatal_exception
+	    raise
+	  end
+	}
+      end
+    end
+
+
+    def add_exports(key, export, njob)
+      @exports_mutex.synchronize do
+	export.no = @no_of_exports
+	@no_of_exports += 1
+	unless expexp = @exports[key]
+	  policy = @opts[:postsuperqueue_queuing_policy]
+	  @exports[key] = expexp = Export.new(policy)
+	  expexp.no = @exports.size - 1
+	  expexp.njob_id = self.class.name
+	  expexp.add_key key
+	  @exports_queue.push [expexp, njob]
+#	  @pre_exports_queue.push [expexp, njob]
+	  expexp.output_no_import = 1
+	end
+	expexp.push_delayed_element {|context|
+	  policy = @opts[:subqueue_queuing_policy]
+	  imp = context.context_eval(%{Import.new(#{policy.inspect})})
+	  imp.no = export.no
+	  imp.add_key(key)
+	  imp.set_log_callback(%q{|n, key| 
+	    Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	  }, nil, __FILE__, __LINE__ - 1)
+
+	  export.output = imp
+	  export.output_no_import = 1
+	  imp
+	}
+#	export.start_export
+      end
+    end
+
+#     def add_exports(key, export, njob)
+#       @exports_mutex.synchronize do
+# 	export.no = @no_of_exports
+# 	@no_of_exports += 1
+# 	unless expexp = @exports[key]
+# 	  policy = @opts[:postsuperqueue_queuing_policy]
+# 	  @exports[key] = expexp = Export.new(policy)
+# 	  expexp.no = @exports.size - 1
+# 	  expexp.add_key key
+# 	  @pre_exports_queue.push [expexp, njob]
+# 	  expexp.output_no_import = 1
+# 	end
+# 	policy = @opts[:subqueue_queuing_policy]
+# 	imp = Import.new(policy)
+# 	imp.no = export.no
+# 	imp.add_key(key)
+# 	imp.set_log_callback do |n| 
+# 	  Log::verbose(self, "IMPORT POP: #{n}")
+# 	end
+
+# 	export.output = imp
+# 	expexp.push imp
+# 	export.output_no_import = 1
+#       end
+#     end
+
+#     def start_watch_all_node_imported
+#       Thread.start do
+# 	@nodes_status_mutex.synchronize do
+# 	  while !all_node_imported?
+# 	    @nodes_status_cv.wait(@nodes_status_mutex)
+# 	  end
+# 	end
+# 	@exports_queue.push nil
+# 	for key, exports in @exports
+# 	  exports.push :END_OF_STREAM
+# 	end
+#       end
+#       nil
+#     end
+
+    def start_watch_all_node_imported
+      Thread.start do
+	# すべての njob がそろうまで待つ
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: S")
+	number_of_nodes
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1")
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 2")
+	# すべての exports がそろうまで待つ
+	@nodes_status_mutex.synchronize do
+	  while !all_node_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+	@exports_queue.push nil
+	
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 4")
+	for key, exports in @exports
+	  exports.push :END_OF_STREAM
+	end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: E")
+      end
+      nil
+    end
+
+    def start_watch_all_node_imported_ORG
+      Thread.start do
+	# すべての njob がそろうまで待つ
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: S")
+	number_of_nodes
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1")
+	# すでに存在するexportsを下流に送る
+	@exports_mutex.synchronize do
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.1")
+	  @pre_exports_queue.push nil
+	  while pair = @pre_exports_queue.pop
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.L")
+	    @exports_queue.push pair
+	  end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.E")
+	end
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 2")
+	# すべての exports がそろうまで待つ
+	@nodes_status_mutex.synchronize do
+	  while !all_node_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 3")
+	# 残りのexportsを下流に送る
+	@pre_exports_queue.push nil
+	while pair = @pre_exports_queue.pop
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 3.L")
+	  @exports_queue.push pair
+	end
+	@exports_queue.push nil
+	
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 4")
+	for key, exports in @exports
+	  exports.push :END_OF_STREAM
+	end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: E")
+      end
+      nil
+    end
+
+
+#     class BPostFilter<BFilter
+#       Controller.def_export self
+
+#       def initialize(controller, opts, block_source)
+# 	super
+# 	@block_source = block_source
+
+# 	@input2node = {}
+# 	@input_queue = PortQueue.new
+# 	@output_queue = PortQueue.new
+#       end
+      
+#       def input=(input)
+# 	@input = input
+# 	start_get_exports
+# 	start
+#       end
+
+#       def start_get_exports
+# 	Thread.start do
+# 	  @input.each_export do |export, node, key|
+# 	    @input2node[export] = node
+# 	    @input_queue.push [export
+# 	  end
+# 	  @input_queue.push nil
+# 	end
+#       end
+
+#       def start
+# 	Thread.start do
+# 	  begin
+# 	    @block.call(@input_queue, @output_queue)
+# 	    @output_queue.push nil
+# 	  ensure
+# 	    if @end_block_source
+# 	      bsource = BSource.new(@end_block_source, @context, self)
+# 	      bsource.evaluate
+# 	    end
+# 	  end
+# 	end
+# 	nil
+#       end
+
+#       def each_export(&block)
+# 	@input.each_export do |export, node, key|
+	  
+# 	for exp in @output_queue
+# 	  node = @input2node[exp]
+# 	  block.call exp, node
+# 	end
+#       end
+#     end
+
+  end
+end

data/lib/fairy/master/c-output-file.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-output"
+
+module Fairy
+  class COutputFile<COutput
+    Controller.def_export self
+
+    def initialize(controller, opts)
+      super
+      @vfile = nil
+
+#      @one_file_by_procs = @opts[:one_file_by_process]
+    end
+
+    def output(vf)
+      @vfile = vf
+    end
+
+    def node_class_name
+      "POutputFile"
+    end
+
+    def njob_creation_params
+      [@vfile]
+    end
+
+# processorで１ファイルにまとめる機能は廃止
+
+#     def create_nodes
+#       return create_node_one_file if @one_file_by_procs
+#       no = 0
+#       @input.each_export do |input_export, input_njob|
+# 	njob = create_and_add_node(input_export, input_njob)
+# 	njob.add_input(nil)
+# 	no += 1
+#       end
+#       self.number_of_nodes = no
+#     end
+
+#     def create_node_one_file
+#       no = 0
+#       input_processors = {}
+#       @input.each_export do |input_export, input_njob|
+# 	if njob = input_processors[input_njob.processor]
+# 	  njob.add_input(input_export)
+# 	else
+# 	  njob = create_and_add_node(input_export, input_njob)
+# 	  input_processors[njob.processor] = njob
+# 	  no += 1
+# 	end
+#       end
+#       for p, njob in input_processors
+# 	njob.add_input(nil)
+#       end
+#       self.number_of_nodes = no
+#     end
+
+
+    def wait_all_output_finished
+      @nodes_status_mutex.synchronize do
+	while !all_node_outputted?
+	  @nodes_status_cv.wait(@nodes_status_mutex)
+	end
+      end
+    end
+
+    def all_node_outputted?
+
+      return false unless @number_of_nodes
+
+      all_outputted = true
+      each_node(:exist_only) do |node|
+	st = @nodes_status[node]
+	all_outputted &&= [:ST_FINISH, :ST_OUTPUT_FINISH].include?(st)
+      end
+      all_outputted
+    end
+
+  end
+end

data/lib/fairy/master/c-output-local-file.rb

@@ -0,0 +1,19 @@
+# encoding: UTF-8
+
+require "fairy/master/c-here"
+
+module Fairy
+  class COutputLocalFile<CHere
+    Controller.def_export(self)
+
+    def node_class_name
+      "POutputLocalFile"
+    end
+
+    def output(filename)
+      @filename = filename
+      backend.output(self)
+    end
+
+  end
+end

data/lib/fairy/master/c-output-null.rb

@@ -0,0 +1,45 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-output"
+
+module Fairy
+  class COutputNull<COutput
+    Controller.def_export self
+
+    def initialize(controller, opts)
+      super
+    end
+
+    def node_class_name
+      "POutputNull"
+    end
+
+#    def njob_creation_params
+#    end
+
+    def number_of_nodes=(no_nodes)
+      super
+    end
+
+    def wait_all_output_finished
+      @nodes_status_mutex.synchronize do
+	while !all_node_outputted?
+	  @nodes_status_cv.wait(@nodes_status_mutex)
+	end
+      end
+    end
+
+    def all_node_outputted?
+      return false unless @nodes_mutex.synchronize{@number_of_nodes}
+
+      each_node(:exist_only) do |node|
+	st = @nodes_status[node]
+	return false unless [:ST_FINISH, :ST_OUTPUT_FINISH].include?(st)
+      end
+      true
+    end
+  end
+end

data/lib/fairy/master/c-output-varray.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-output"
+
+require "fairy/share/varray"
+
+module Fairy
+  class COutputVArray<COutput
+    Controller.def_export self
+
+    def initialize(controller, opts)
+      super
+      @varray = VArray.new(nil)
+
+      @node_id = 0
+    end
+
+    attr_reader :varray
+
+    def node_class_name
+      "POutputVArray"
+    end
+
+    def njob_creation_params
+      @node_id += 1
+      [@node_id-1]
+#      []
+    end
+
+    def number_of_nodes=(no_nodes)
+      super
+      @varray.arrays_size = no_nodes
+    end
+
+    def wait_all_output_finished
+      @nodes_status_mutex.synchronize do
+	while !all_node_outputted?
+	  @nodes_status_cv.wait(@nodes_status_mutex)
+	end
+      end
+    end
+
+    def all_node_outputted?
+      return false unless @nodes_mutex.synchronize{@number_of_nodes}
+
+      each_node(:exist_only) do |node|
+	st = @nodes_status[node]
+	return false unless [:ST_FINISH, :ST_OUTPUT_FINISH].include?(st)
+      end
+      true
+    end
+
+  end
+end