fairy 0.6.0

data/lib/fairy/master/c-find.rb

@@ -0,0 +1,138 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "forwardable"
+
+require "deep-connect"
+
+require "fairy/master/c-io-filter"
+
+module Fairy
+  class CFind<CIOFilter
+    extend Forwardable
+
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+
+      @block_source = block_source
+
+      @clocal_find = CLocalFind.new(controller, opts, block_source)
+      @cfind_result = CFindResult.new(controller, opts, self)
+
+    end
+
+    def_delegator :@cfind_result, :value
+    def_delegator :@cfind_result, :output=
+
+    def input=(input)
+      @clocal_find.input = input
+      @cfind_result.input = @clocal_find
+    end
+
+    def update_find
+      @clocal_find.break_running
+    end
+
+    class CLocalFind<CIOFilter
+      def initialize(controller, opts, block_source)
+	super
+	@block_source = block_source
+
+	@no = 0
+	@first_node = nil
+	@first_node_mutex = Mutex.new
+      end
+
+      def node_class_name
+	"PLocalFind"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+      
+      def each_assigned_filter(&block)
+	super
+
+	@first_node.export.output_no_import = @no
+      end
+
+      def each_export_by(njob, mapper, &block)
+	@first_node_mutex.synchronize do
+	  @no += 1
+	  if @first_node
+	    njob.export.output = @first_node.export.output
+	    njob.export.no = @no - 1
+	  else
+	    super
+	    @first_node = njob
+	  end
+	end
+      end
+
+      def bind_export(exp, imp)
+	# do nothing.
+      end
+
+#       def each_export(&block)
+# 	no = 0
+# 	first_node = nil
+# 	each_node do |node|
+# 	  no += 1
+# 	  if first_node
+# 	    node.export.output = first_node.export.output
+# 	  else
+# 	    first_node = node
+# 	    block.call node.export, node
+# 	  end
+# 	end
+# 	first_node.export.output_no_import = no
+#       end
+
+#       def find_break
+# 	# create node 中ならそれをとめる
+# 	break_create_node
+# 	# 各tasklettをとめる
+# 	each_node do |tasklet|
+# 	  tasklet.find_break
+# 	end
+#       end
+
+    end
+
+    class CFindResult<CIOFilter
+      def initialize(controller, opts, bfind)
+	super
+	@bfind = bfind
+
+	@find_mutex = Mutex.new
+	@findp = false
+      end
+
+      def node_class_name
+	"PFindResult"
+      end
+
+      def njob_creation_params
+	[]
+      end
+
+      def value
+	each_node{|node| return node.value}
+      end
+
+      def update_find
+	@find_mutex.synchronize do
+	  if !@findp
+	    @findp = true
+	    @bfind.update_find
+	  end
+	end
+      end
+    end
+  end
+end

data/lib/fairy/master/c-group-by.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+require "fairy/master/c-basic-group-by"
+
+module Fairy
+  class CGroupBy<CBasicGroupBy
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+
+      @hash_seed = controller.hash_seed
+    end
+
+    attr_reader :hash_seed
+
+    def node_class_name
+      "PGroupBy"
+    end
+
+    class CPostFilter<CIOFilter
+      Controller.def_export self
+
+      def initialize(controller, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def node_class_name
+	"PGroupBy::PPostFilter"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+
+      def create_import(processor)
+	policy = @opts[:postfilter_prequeuing_policy]
+	policy ||= @opts[:prequeuing_policy]
+	
+	processor.create_import(policy)
+      end
+    end
+
+#   class BPostAfterModFilter<BFilter
+#     Controller.def_export self
+
+#     def initialize(controller, opts, block_source)
+#       super
+#       @block_source = block_source
+#     end
+
+#     def node_class_name
+#       "NPostAfterModFilter"
+#     end
+#   end
+
+  end
+end

data/lib/fairy/master/c-here.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+require "fairy/share/port"
+
+
+module Fairy
+  class CHere<CIOFilter
+    Controller.def_export self
+
+    def node_class_name
+      "PHere"
+    end
+
+    def create_and_add_node(ntask, mapper, opts={})
+      node = create_node(ntask) {|node|
+	if opts[:init_njob]
+	  opts[:init_njob].call(node)
+	end
+	mapper.bind_input(node)
+#	exp = node.start_export
+      }
+      node
+    end
+
+    def each(&block)
+      policy = @opts[:prequeuing_policy]
+      each_node do |node|
+	node.start_export
+	import = Import.new(policy)
+	import.set_log_callback do |n, key| 
+	  Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	end
+	import.no_import = 1
+	node.export.output = import
+	import.each do |e|
+	  block.call e
+	end
+      end
+    end
+    DeepConnect.def_method_spec(self, "each(){DVAL}")
+
+    def each_buf(&block)
+      threshold = @opts[:pool_threshold]  || CONF.HERE_POOL_THRESHOLD
+      chunk = []
+
+      policy = @opts[:prequeuing_policy]
+      each_node do |node|
+	node.start_export
+	import = Import.new(policy)
+	import.set_log_callback do |n, key| 
+	  Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	end
+	import.no_import = 1
+	node.export.output = import
+
+	import.each do |e|
+	  #[REQ#89]
+
+	  chunk.push e
+	  if chunk.size > threshold
+	    block.call chunk
+	    chunk.clear
+	  end
+	end
+      end
+      if !chunk.empty?
+	block.call chunk
+      end
+    end
+    DeepConnect.def_method_spec(self, "each_buf(){DVAL}")
+  end
+end
+
+

data/lib/fairy/master/c-inject.rb

@@ -0,0 +1,119 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "forwardable"
+
+require "deep-connect"
+
+require "fairy/master/c-io-filter"
+
+module Fairy
+  class CInject<CIOFilter
+    extend Forwardable
+
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+
+      @block_source = block_source
+
+      @clocal_inject = CLocalInject.new(controller, opts, block_source)
+      @cwide_inject = CWideInject.new(controller, opts, block_source)
+
+    end
+
+    def_delegator :@cwide_inject, :value
+    def_delegator :@cwide_inject, :output=
+    def_delegator :@cwide_inject, :each_assigned_filter
+    def_delegator :@cwide_inject, :each_export_by
+    def_delegator :@cwide_inject, :bind_export
+    
+
+    def input=(input)
+      @clocal_inject.input = input
+      @cwide_inject.input = @clocal_inject
+    end
+
+    class CLocalInject<CIOFilter
+      def initialize(controller, opts, block_source)
+	super
+	@block_source = block_source
+
+	@no = 0
+	@first_node = nil
+	@first_node_mutex = Mutex.new
+      end
+
+      def node_class_name
+	"PLocalInject"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+
+      def each_assigned_filter(&block)
+	super
+
+	@first_node.export.output_no_import = @no
+      end
+
+      def each_export_by(njob, mapper, &block)
+	@first_node_mutex.synchronize do
+	  @no += 1
+	  if @first_node
+	    njob.export.output = @first_node.export.output
+	    njob.export.no = @no - 1
+	  else
+	    super
+	    @first_node = njob
+	  end
+	end
+      end
+
+      def bind_export(exp, imp)
+	# do nothing.
+      end
+
+
+#       def each_export(&block)
+
+# 	no = 0
+# 	first_node = nil
+# 	each_node do |node|
+# 	  no += 1
+# 	  if first_node
+# 	    node.export.output = first_node.export.output
+# 	  else
+# 	    first_node = node
+# 	    block.call node.export, node
+# 	  end
+# 	end
+# 	first_node.export.output_no_import = no
+#       end
+
+    end
+
+    class CWideInject<CIOFilter
+      def initialize(controller, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def node_class_name
+	"PWideInject"
+      end
+
+      def njob_creation_params
+	[@block_source]
+      end
+
+      def value
+	each_node{|node| return node.value}
+      end
+    end
+  end
+end

data/lib/fairy/master/c-input-file.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "uri"
+
+require "fairy/master/c-input"
+require "fairy/share/vfile"
+require "fairy/share/file-place"
+
+module Fairy
+  class CInputFile<CInput
+    Controller.def_export self
+
+    def CInputFile.open(controller, opts, descripter)
+      bfile = CInputFile.new(controller, opts)
+      bfile.open(desctipter)
+      bfile
+    end
+    DeepConnect.def_single_method_spec(self, "REF open(REF, VAL, VAL)")
+
+    def node_class_name
+      "PInputFile"
+    end
+
+    def open(vf)
+      @vfile = vf
+      @cfile_place = CFilePlace.new(@vfile)
+
+      start_create_nodes
+    end
+    DeepConnect.def_method_spec(self, "REF open(DVAL)")
+
+    def input
+      @cfile_place
+    end
+
+#     def create_and_add_node(processor, mapper)
+#       node = super
+#       node.open(mapper.input.path)
+#       node
+#     end
+
+  end
+end