fairy 0.6.0

data/lib/fairy/node/p-merger-group-by.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-basic-group-by"
+
+module Fairy
+  class PMergeGroupBy<PBasicGroupBy
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+    end
+
+#     class NPostFilter<NSingleExportFilter
+#       Processor.def_export self
+
+#       def initialize(processor, bjob, opts, block_source)
+# 	super
+# 	@block_source = block_source
+#       end
+
+#       def start
+# 	basic_start do
+# 	  @key_value = {}
+# 	  @hash_proc = BBlock.new(@block_source, @context, self)
+
+# 	  @import.each do |e|
+# 	    key = key(e)
+# 	    @key_value[e] = [] unless @key_value.key?(e)
+# 	    @key_value[e].push e
+# 	  end
+# 	  for key, values in @key_value
+# 	    @export.push [key, values]
+# 	  end
+# 	end
+#       end
+
+#       def key(e)
+# 	@hash_proc.yield(e)
+#       end
+#     end
+  end
+end
+
+

data/lib/fairy/node/p-output-file.rb

@@ -0,0 +1,104 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "uri"
+
+require "fairy/node/p-io-filter"
+
+module Fairy
+  class POutputFile<PIOFilter
+    Processor.def_export self
+    
+    ST_OUTPUT_FINISH = :ST_OUTPUT_FINISH
+
+    def initialize(id, ntask, bjob, opt, vf)
+      super
+      @vfile = vf
+
+#      @imports = Queue.new
+    end
+
+    def input=(input)
+      super
+      start
+    end
+
+#     def add_input(input)
+#       unless input
+# 	@imports.push nil
+# 	return self
+#       end
+#       policy = @opts[:prequeuing_policy]
+#       import = Import.new(policy)
+#       import.add_key(input.key)
+#       input.output = import
+#       import.set_log_callback do |n| 
+# 	Log::verbose(self, "IMPORT POP: #{n}")
+#       end
+#       @imports.push import
+#       self
+#     end
+
+    def basic_start(&block)
+      Log::debug(self, "START")
+      output_uri = gen_real_file_name
+      @vfile.set_real_file(no, output_uri)
+
+      Log::debug(self, "write real file: #{output_uri}")
+      begin
+	output_file = URI.parse(output_uri).path
+      rescue
+	Log::debug_exception(self)
+	raise
+      end
+
+      unless File.exist?(File.dirname(output_file))
+	create_dir(File.dirname(output_file))
+      end
+
+      File.open(output_file, "w") do |io|
+	Log::debug(self, "start write real file: #{output_uri}")
+	@input.each do |l|
+	  io.puts l
+	end
+	Log::debug(self, "finish write real file: #{output_uri}")
+      end
+      self.status = ST_OUTPUT_FINISH
+    end
+
+    def create_dir(path)
+      unless File.exist?(File.dirname(path))
+	create_dir(File.dirname(path))
+      end
+      begin
+	Dir.mkdir(path)
+      rescue Errno::EEXIST
+	# 無視
+      end
+    end
+
+    IPADDR_REGEXP = /::ffff:([0-9]+\.){3}[0-9]+|[0-9a-f]+:([0-9a-f]*:)[0-9a-f]*/
+
+    def gen_real_file_name
+      host= processor.addr
+      root = CONF.VF_ROOT
+      prefix = CONF.VF_PREFIX
+      base_name = @vfile.base_name
+      no = @input.no
+      
+
+      if IPADDR_REGEXP =~ host
+	begin
+	  host = Resolv.getname(host)
+	rescue
+	  # ホスト名が分からない場合 は そのまま ipv6 アドレスにする
+	  host = "[#{host}]"
+	end
+      end
+      
+      format("file://#{host}#{root}/#{prefix}/#{base_name}-%03d", no)
+    end
+  end
+end

data/lib/fairy/node/p-output-local-file.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "uri"
+
+require "fairy/node/p-here"
+
+module Fairy
+  class POutputLocalFile<PHere
+    Processor.def_export self
+  end
+end

data/lib/fairy/node/p-output-null.rb

@@ -0,0 +1,32 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+
+module Fairy
+  class POutputNull<PIOFilter
+    Processor.def_export self
+    
+    ST_OUTPUT_FINISH = :ST_OUTPUT_FINISH
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, REF, VAL, REF)")
+
+    def initialize(id, ntask, bjob, opt)
+      super
+    end
+
+    def input=(input)
+      super
+      start
+    end
+
+    def basic_start(&block)
+      for l in @input
+	l
+      end
+      self.status = ST_OUTPUT_FINISH
+    end
+  end
+end

data/lib/fairy/node/p-output-varray.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+
+require "fairy/share/varray"
+
+module Fairy
+  class POutputVArray<PIOFilter
+    Processor.def_export self
+    
+    ST_OUTPUT_FINISH = :ST_OUTPUT_FINISH
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, REF, VAL, REF)")
+
+#    def initialize(processor, bjob, opt, varray, idx)
+    def initialize(id, ntask, bjob, opt, idx)
+      super
+      @no_in_bjob = idx
+    end
+
+    def input=(input)
+      super
+      start
+    end
+
+
+    def basic_start(&block)
+      array = []
+      for l in @input
+	array.push l
+      end
+	
+      processor.register_varray_element(array)
+      @bjob.varray.arrays_put(@no_in_bjob, array)
+      self.status = ST_OUTPUT_FINISH
+    end
+  end
+end

data/lib/fairy/node/p-seg-join.rb

@@ -0,0 +1,82 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PSegJoin<PSingleExportFilter
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+
+      @join_imports = nil
+      @join_imports_mutex = Mutex.new
+      @join_imports_cv = ConditionVariable.new
+    end
+
+    def join_imports
+      @join_imports_mutex.synchronize do
+	while !@join_imports
+	  @join_imports_cv.wait(@join_imports_mutex)
+	end
+      end
+      @join_imports
+    end
+
+    DeepConnect.def_method_spec(self, "VAL join_imports")
+
+    def join_inputs=(jinputs)
+      policy = @opts[:prequeuing_policy]
+      @join_imports_mutex.synchronize do
+	@join_imports = jinputs.collect{|jinput| 
+	  if jinput
+	    import = Import.new(policy)
+	    import.no = jinput.no
+	    import.add_key(jinput.key)
+	    import.set_log_callback do |n, key| 
+	      Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	    end
+
+	    import
+	  else
+	    nil
+	  end
+	}
+	@join_imports_cv.broadcast
+      end
+      @join_imports
+    end
+
+    DeepConnect.def_method_spec(self, :rets => "VAL", :method => :join_inputs=, :args => "VAL")
+
+    def basic_each(&block)
+      @map_proc = BBlock.new(@block_source, @context, self)
+      arg = [@input]
+      arg.push *join_imports
+      arg.push block
+	  
+      @map_proc.yield(*arg)
+    end
+
+
+#     def start
+#       super do
+# 	@map_proc = BBlock.new(@block_source, @context, self)
+# 	arg = [@import]
+# 	arg.push *join_imports
+# 	arg.push @export
+	  
+# Log::debug(self, "START")
+# #Log::debug(self, @block_source.source)
+# #	@map_proc.yield(@import, *join_imports, @export)
+# 	@map_proc.yield(*arg)
+# Log::debug(self, "END")
+#       end
+#     end
+  end
+end

data/lib/fairy/node/p-seg-map.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PSegMap<PSingleExportFilter
+    Processor.def_export self
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, REF, VAL, VAL)")
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+#      @map_proc = eval("proc{#{@block_source}}", TOPLEVEL_BINDING)
+#      @map_proc = @context.create_proc(@block_source)
+    end
+
+#    def start
+#      super do
+#	@map_proc = BBlock.new(@block_source, @context, self)
+#	@map_proc.yield(@import, @export)
+#      end
+#    end
+
+    def basic_each(&block)
+      @map_proc = BBlock.new(@block_source, @context, self)
+      @map_proc.yield(@input, block)
+    end
+
+    # basic_nextがない
+
+  end
+end

data/lib/fairy/node/p-seg-split.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+
+module Fairy
+  class PSegSplit<PIOFilter
+    Processor.def_export self
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, VAL)")
+
+    def initialize(id, ntask, bjob, opts, n)
+      super
+      @no_split = n
+
+      policy = @opts[:postqueuing_policy]
+      @exports = @no_split.times.collect{Export.new(policy)}
+    end
+
+    attr_reader :exports
+    DeepConnect.def_method_spec(self, "VAL exports")
+
+    def start_export
+      Log::debug(self, "START_EXPORT")
+#      return unless @status == ST_WAIT_IMPORT
+      start do
+	begin
+	  @input.each_slice(@no_split) do |ll|
+	    if ll.size < @no_split
+	      ll.fill(0, @no_split){|idx| ll[idx] ||= END_OF_STREAM}
+	    end
+	    @exports.zip(ll) do |exp, l|
+	      exp.push l
+	    end
+	  end
+	ensure
+	  @exports.each{|exp| exp.push END_OF_STREAM}
+	end
+      end
+    end
+
+#     def start
+#       super do
+# 	begin
+# 	  @import.each_slice(@no_split) do |ll|
+# 	    if ll.size < @no_split
+# 	      ll.fill(0, @no_split){|idx| ll[idx] ||= END_OF_STREAM}
+# 	    end
+# 	    @exports.zip(ll) do |exp, l|
+# 	      exp.push l
+# 	    end
+# 	  end
+# 	ensure
+# 	  @exports.each{|exp| exp.push END_OF_STREAM}
+# 	end
+#       end
+#     end
+  end
+end

data/lib/fairy/node/p-seg-zip.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PSegZip<PSingleExportFilter
+    Processor.def_export self
+
+#    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL, VAL, VAL)")
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+#      @map_proc = eval("proc{#{@block_source}}", TOPLEVEL_BINDING)
+#      @map_proc = @context.create_proc(@block_source)
+
+      @zip_imports = nil
+      @zip_imports_mutex = Mutex.new
+      @zip_imports_cv = ConditionVariable.new
+    end
+
+    def zip_imports
+      @zip_imports_mutex.synchronize do
+	while !@zip_imports
+	  @zip_imports_cv.wait(@zip_imports_mutex)
+	end
+      end
+      @zip_imports
+    end
+
+    DeepConnect.def_method_spec(self, "VAL zip_imports")
+
+    def zip_inputs=(zinputs)
+
+      policy = @opts[:prequeuing_policy]
+      # 仮
+      @zip_imports_mutex.synchronize do
+	@zip_imports = zinputs.collect{|zinput| 
+	  import = Import.new(policy)
+	  import.no = zinput.no
+	  import.add_key(zinput.key)
+	  import.set_log_callback do |n, key| 
+	    Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	  end
+
+	  import
+	}
+	@zip_imports_cv.broadcast
+      end
+      @zip_imports
+    end
+    DeepConnect.def_method_spec(self, :rets => "VAL", :method => :zip_inputs=, :args => "VAL")
+
+    def basic_each(&block)
+      @map_proc = BBlock.new(@block_source, @context, self)
+      @input.each do |e|
+	zips = zip_imports.collect{|import| import.pop}
+	if Import::CTLTOKEN_NULLVALUE === (v = @map_proc.yield(e, *zips))
+	  next
+	end
+	block.call v
+      end
+    end
+
+#     def start
+#       super do
+# 	@map_proc = BBlock.new(@block_source, @context, self)
+# 	@import.each do |e|
+# 	  zips = zip_imports.collect{|import| import.pop}
+# 	  @export.push @map_proc.yield(e, *zips)
+# 	end
+#       end
+#     end
+  end
+end