fairy 0.6.0

data/lib/fairy/master/c-output.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class COutput<CFilter
+    include CInputtable
+
+    def input=(input)
+      @input = input
+      input.output = @input
+      super
+    end
+
+  end
+end

data/lib/fairy/master/c-seg-join.rb

@@ -0,0 +1,141 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CSegJoin<CIOFilter
+    Controller.def_export self
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, VAL, REF)")
+
+    def initialize(controller, opts, others, block_source)
+      super
+      @others = others
+      @block_source = block_source
+
+      @exports = {}
+      @others_status = {}
+      @exports_mutex = Mutex.new
+      @exports_cv = ConditionVariable.new
+
+      @key_proc
+      init_key_proc
+    end
+
+    def init_key_proc
+      case join_by
+      when :ORDER
+	@key_proc = proc{|input| input.no}
+      when :KEY
+	@key_proc = proc{|input| input.key}
+      else
+	ERR::Raise ERR::UnrecoginizedOption, join_by
+      end
+    end
+
+    def join_by
+      by = @opts[:by]
+      return :ORDER unless by
+      by.to_s.upcase.intern
+    end
+
+    def node_class_name
+      "PSegJoin"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+
+    def start_create_nodes
+      Log::debug self, "START_CREATE_NODES: #{self}"
+      @others.each do |other|
+	Thread.start do
+	  other.each_assigned_filter do |input_filter|
+	    @exports_mutex.synchronize do
+	      unless exps = @exports[@key_proc.call(input_filter)]
+		exps = @exports[@key_proc.call(input_filter)] = {}
+		@exports_cv.broadcast
+	      end
+	      exp = input_filter.start_export
+	      exps[other] = exp
+	    end
+	  end
+	  @exports_mutex.synchronize do
+	    @others_status[other] = true
+	    @exports_cv.broadcast
+	  end
+	end
+      end
+      super
+    end
+
+    def create_and_add_node(ntask, mapper, opts={})
+      node = create_node(ntask) {|node|
+	if opts[:init_njob]
+	  opts[:init_njob].call(node)
+	end
+	mapper.bind_input(node)
+
+	key = @key_proc.call(node)
+	exps = nil
+	@exports_mutex.synchronize do
+	  while !(exps = other_filter_of(key))
+	    @exports_cv.wait(@exports_mutex)
+	  end
+	end
+ 	node.join_inputs = exps
+ 	exps.zip(node.join_imports) do |other, import|
+	  other.output = import
+	  import.no_import = 1
+	end
+      }
+      node
+    end
+
+    class NoAllFilter<Exception;end
+
+    def other_filter_of(key)
+      begin
+	@others.collect do |o| 
+	  unless exp = @exports[key][o]
+	    unless @other_status[o]
+	      raise NoAllFilter
+	    end
+	  end
+	  exp
+	end
+      rescue NoAllFilter
+	return nil
+      rescue
+	return nil
+      end
+    end
+
+    def break_running
+      super
+      @others.each{|other| Thread.start{other.break_running}}
+    end
+
+    class CPreSegJoinFilter<CIOFilter
+      Controller.def_export self
+
+      def initialize(controller, opts)
+	super
+      end
+
+      def node_class_name
+	"PIdentity"
+      end
+
+      def njob_creation_params
+	[]
+      end
+    end
+
+  end
+end

data/lib/fairy/master/c-seg-map.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CSegMap<CIOFilter
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def node_class_name
+      "PSegMap"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+  end
+end

data/lib/fairy/master/c-seg-shuffle.rb

@@ -0,0 +1,87 @@
+# encoding: UTF-8
+
+require "delegate"
+
+require "fairy/master/c-filter"
+require "fairy/master/c-io-filter"
+
+
+module Fairy
+  class CSegShuffle<CIOFilter
+    Controller.def_export self
+
+#    include BInputtable
+
+    def initialize(controller, opts, block_source)
+      super
+#      @block = @context.create_proc(@block_source.source)
+
+      @node2input = {}
+      @input2node = {}
+      @input_queue = PortQueue.new
+      @output_queue = PortQueue.new
+
+      @block_source = block_source
+      @begin_block_source = nil
+      if @opts[:BEGIN]
+	@begin_block_source = @opts[:BEGIN]
+      end
+      @end_block_source = nil
+      if @opts[:END]
+	@end_block_source = @opts[:END]
+      end
+
+    end
+
+    def node_class_name
+      "PIdentity"
+    end
+
+    def njob_creation_params
+      []
+    end
+
+    def each_assigned_filter(&block)
+      each_node do |node|
+	@input_queue.push node
+      end
+      @input_queue.push nil
+
+      if @begin_block_source
+	bsource = BScript.new(@begin_block_source, @context, self)
+	bsource.evaluate
+      end
+      @block = BBlock.new(@block_source, @context, self)
+      begin
+	@block.call(@input_queue, @output_queue)
+	@output_queue.push nil
+      ensure
+	if @end_block_source
+	  bsource = BSource.new(@end_block_source, @context, self)
+	  bsource.evaluate
+	end
+      end
+
+      no = 0
+      @output_queue.each do |node|
+	node.no = no
+	no += 1
+	block.call node
+      end
+    end
+
+    class PortQueue<DelegateClass(Queue)
+      include Enumerable
+
+      def initialize
+	super(Queue.new)
+      end
+
+      def each
+	while e = pop
+	  yield e
+	end
+      end
+    end
+  end
+end

data/lib/fairy/master/c-seg-split.rb

@@ -0,0 +1,110 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CSegSplit<CIOFilter
+    Controller.def_export self
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, VAL)")
+
+    def initialize(controller, opts, n)
+      super
+      @no_split = n
+
+      @no_of_exports = 0
+      @no_of_exports_mutex = Mutex.new
+
+#      @exports = []
+#      @exports_mutex = Mutex.new
+#      @exports_cv = ConditionVariable.new
+
+#      @export2njob = {}
+
+#      @exports_queue = Queue.new
+    end
+
+#     def start_create_nodes
+#       super
+#       Thread.start{appear_njob}
+#     end
+
+#     def appear_njob
+# Log::debug(self, "APPEAR_NJOB:S")
+#       loop do
+# 	@nodes_mutex.synchronize do
+# 	  while @nodes_for_next_filters.empty?
+# 	    @nodes_cv.wait(@nodes_mutex)
+# 	  end
+# Log::debug(self, "APPEAR_NJOB:1")
+# 	  njob = @nodes_for_next_filters.shift
+# Log::debug(self, "APPEAR_NJOB:2")
+# 	  unless njob
+# Log::debug(self, "APPEAR_NJOB:E")
+# 	    @exports_queue.push nil
+# 	    return
+# 	  end
+
+# Log::debug(self, "APPEAR_NJOB:3")
+# 	  njob.exports.each do |exp|
+# 	    exp.no = @no_of_exports
+# 	    #	  exp.key = njob.key
+# 	    @no_of_exports += 1
+
+# 	    @export2njob[exp] = njob
+# 	    @exports_queue.push exp
+# 	  end
+# Log::debug(self, "APPEAR_NJOB:4")
+# 	end
+#       end
+#     end
+
+#     def next_filter(mapper)
+#       @exports_queue.pop
+#     end
+
+#     def start_export(njob)
+# Log::debug(self, "START_EXPORT:SS")
+
+# Log::debug(self, "START_EXPORT: #{exp.class} #{@export2njob[exp].class}")
+#       njob.start_export
+# Log::debug(self, "START_EXPORT:EE")
+#       exp
+#     end
+
+    def each_export_by(njob, mapper, &block)
+      njob.exports.each do |exp|
+	@no_of_exports_mutex.synchronize do
+	  exp.no = @no_of_exports
+	  #  exp.key = njob.key
+	  @no_of_exports += 1
+	end
+	
+	block.call exp
+      end
+    end
+
+#     def each_export(&block)
+#       each_node do |node|
+# 	for exp in node.exports
+# 	  exp.no = @no_of_exports
+# 	  @no_of_exports += 1
+# 	  block.call exp, node
+# 	  exp.output_no_import = 1
+# 	end
+#       end
+#     end
+
+    def node_class_name
+      "PSegSplit"
+    end
+    
+    def njob_creation_params
+      [@no_split]
+    end
+  end
+end

data/lib/fairy/master/c-seg-zip.rb

@@ -0,0 +1,132 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CSegZip<CIOFilter
+    Controller.def_export self
+
+    ZIP_BY_SEGMENT = :ZIP_BY_SEGMENT
+
+    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, VAL, REF)")
+
+    def initialize(controller, opts, others, block_source)
+      super
+      @others = others
+      @block_source = block_source
+
+      #@exports = [{o=>filter, ...}, ...]
+      @exports = []
+      @others_status = {}
+      @exports_mutex = Mutex.new
+      @exports_cv = ConditionVariable.new
+    end
+
+    def opt_zip_by_substream?
+      @opts[ZIP_BY_SEGMENT]
+    end
+
+    def node_class_name
+      "PSegZip"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+
+    def start_create_nodes
+      Log::debug self, "START_CREATE_NODES: #{self}"
+      @others.each do |other|
+	Thread.start do
+	  other.each_assigned_filter do |input_filter|
+	    @exports_mutex.synchronize do
+	      unless exps = @exports[input_filter.no]
+		exps = @exports[input_filter.no] = {}
+		@exports_cv.broadcast
+	      end
+	      exp = input_filter.start_export
+	      exps[other] = exp
+	    end
+	  end
+	  @exports_mutex.synchronize do
+	    @others_status[other] = true
+	    @exports_cv.broadcast
+	  end
+	end
+      end
+      super
+    end
+
+    class NoAllFilter<Exception;end
+
+    def create_and_add_node(ntask, mapper, opts={})
+      unless opt_zip_by_substream?
+ 	ERR::Raise ERR::NoImplement, "except zip_by_segment"
+      end
+
+      node = create_node(ntask) {|node|
+	if opts[:init_njob]
+	  opts[:init_njob].call(node)
+	end
+	mapper.bind_input(node)
+
+	no = node.no
+	exps = nil
+	@exports_mutex.synchronize do
+	  while !(exps = other_filter_of(no))
+	    @exports_cv.wait(@exports_mutex)
+	  end
+	end
+ 	node.zip_inputs = exps
+ 	exps.zip(node.zip_imports) do |other, import| 
+	  other.output = import
+	  import.no_import = 1
+	end
+      }
+      node
+    end
+
+    def other_filter_of(no)
+      begin
+	@others.collect do |o| 
+	  unless exp = @exports[no][o]
+	    unless @other_status[o]
+	      raise NoAllFilter
+	    end
+	  end
+	  exp
+	end
+      rescue NoAllFilter
+	return nil
+      rescue
+	return nil
+      end
+    end
+
+    def break_running
+      super
+      @others.each{|others| Thread.start{others.break_running}}
+    end
+
+
+    class CPreSegZipFilter<CIOFilter
+      Controller.def_export self
+
+      def initialize(controller, opts)
+	super
+      end
+
+      def node_class_name
+	"PIdentity"
+      end
+
+      def njob_creation_params
+	[]
+      end
+    end
+  end
+end