fairy 0.6.0

data/lib/fairy/node/p-select.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PSelect<PSingleExportFilter
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+#     def start
+#       super do
+# 	@map_proc = BBlock.new(@block_source, @context, self)
+# 	@import.each do |e|
+# 	  if @map_proc.yield(e)
+# 	    @export.push e
+# 	  end
+# 	end
+#       end
+#     end
+
+    def basic_each(&block)
+      @map_proc = BBlock.new(@block_source, @context, self)
+      @input.each do |e|
+#	if !(v = @map_proc.yield(e)) || v.kind_of?(Import::CTLTOKEN_NULLVALUE)
+	if !(v = @map_proc.yield(e)) || Import::CTLTOKEN_NULLVALUE === v
+ 	  next
+	end
+	block.call e
+      end
+    end
+  end
+end

data/lib/fairy/node/p-single-exportable.rb

@@ -0,0 +1,90 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+ 
+module Fairy
+  module PSingleExportable
+    include Enumerable
+
+    END_OF_STREAM = PFilter::END_OF_STREAM
+
+    ST_WAIT_EXPORT_FINISH = :ST_WAIT_EXPORT_FINISH
+    ST_EXPORT_FINISH = :ST_EXPORT_FINISH
+
+    def initialize(id, ntask, bjob, opts=nil, *rests)
+      super
+    end
+
+    attr_reader :export
+
+#    def no=(no)
+#      super
+#      @export.no = no
+#    end
+
+# とりあえず
+#     def start(&block)
+#       super do
+# 	begin
+# 	  if @import
+# 	    @export.add_key(@import.key)
+# 	  end
+# 	  block.call
+# 	ensure
+# 	  @export.push END_OF_STREAM
+# 	  wait_export_finish
+# 	end
+#       end
+#     end
+
+    def start_export
+      Log::debug(self, "START_EXPORT")
+
+      policy = @opts[:postqueuing_policy]
+      @export = Export.new(policy)
+      @export.njob_id = @id
+      @export.no = @no
+      @export.key = @key
+
+      start do
+	each{|e| @export.push e}
+	@export.push END_OF_STREAM
+      end
+
+      @export
+    end
+
+    def start(&block)
+      super do
+ 	begin
+ 	  block.call
+ 	ensure
+# 	  @export.push END_OF_STREAM
+ 	end
+      end
+    end
+
+    def terminate
+      @wait_cv = @terminate_mon.new_cv
+      wait_export_finish
+      super
+    end
+
+    def wait_export_finish
+      self.status = ST_WAIT_EXPORT_FINISH
+      @export.fib_wait_finish(@wait_cv)
+      self.status = ST_EXPORT_FINISH
+    end
+  end
+
+  class PSingleExportFilter<PIOFilter
+    include PSingleExportable
+  end
+
+  class PSingleExportInput<PIOFilter
+    include PSingleExportable
+  end
+end

data/lib/fairy/node/p-sort.rb

@@ -0,0 +1,195 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-basic-group-by"
+
+module Fairy
+  module PSort
+    class PPreSort<PBasicGroupBy
+      Processor.def_export self
+
+      ST_ALL_IMPORTED = :ST_ALL_IMPORTED
+      ST_WAIT_EXPORT_FINISH = :ST_WAIT_EXPORT_FINISH
+      ST_EXPORT_FINISH = :ST_EXPORT_FINISH
+
+      def initialize(id, ntask, bjob, opts, block_source)
+	super
+	@block_source = block_source
+
+	@exports = []
+	def @exports.each_pair(&block)
+	  each_with_index do |item, idx| 
+	    block.call(idx, item) if item
+	  end
+	end
+	@exports_queue = Queue.new
+	
+	@counter = []
+
+	@pvs = nil
+	if @opts[:pvs]
+	  @pvs = @opts[:pvs].dc_deep_copy
+	end
+
+	#start_watch_exports
+      end
+
+      def add_export(key, export)
+	@exports[key] = export
+	#      @exports_queue.push [key, export]
+	# [BUG#171]同期処理でないとまずい.
+	@bjob.add_exports(key, export, self)
+      end
+
+      def start_export
+	Log::debug(self, "START_EXPORT")
+
+	start do
+	  sample_line_no = @opts[:sampling_max]
+	  sample_line_no ||= CONF.SORT_SAMPLING_MAX
+
+	  hash_opt = @opts[:cmp_optimize]
+	  hash_opt = CONF.SORT_CMP_OPTIMIZE if hash_opt.nil?
+	  
+	  if hash_opt
+	    @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
+	  else
+	    @key_proc = BBlock.new(@block_source, @context, self)
+	  end
+	  
+	  buf = []
+	  no = 0
+	  begin
+	    if @pvs
+	      sampling = false
+Log::debugf(self, "%s", @pvs.inspect)
+	      init_exports
+	    elsif self.no == 0
+	      sampling = true
+	    else
+	      sampling = false
+	      @pvs = @bjob.get_pvs
+Log::debugf(self, "%s", @pvs.inspect)
+	      init_exports
+	    end
+	      
+	    @input.each do |e|
+	      if sampling
+		no += 1
+		buf.push e
+		if no >= sample_line_no
+		  sampling = false
+		  @pvs = @bjob.get_pvs(buf)
+Log::debugf(self, "%s", @pvs.inspect)
+		  init_exports
+		  buf.each{|e| hashing(e)}
+		end
+	      else
+		hashing(e)
+	      end
+	    end
+	    if sampling
+	      @pvs = @bjob.get_pvs(buf)
+Log::debugf(self, "%s", @pvs.inspect)
+	      init_exports
+	      buf.each{|e| hashing(e)}
+	    end
+	  rescue
+	    Log::debug_exception(self)
+	    raise
+	  ensure
+	    @exports_queue.push nil
+	    @exports.each_pair do |key, export| 
+	      next unless export
+	      Log::debug(self, "G0 #{key} => #{@counter[key]}")	    
+	      export.push END_OF_STREAM
+	    end
+	  end
+	end
+      end
+
+      def init_exports
+	policy = @opts[:postqueuing_policy]
+	(@pvs.size+1).times do |idx|
+	  export = Export.new(policy)
+	  @exports[idx] = export
+	  
+	  export.njob_id = @id
+	  export.no = idx
+	  export.add_key(idx)
+	  add_export(idx, export)
+	  @counter[idx] = 0
+	end
+      end
+
+      def hashing(e)
+	if Import::CTLTOKEN_NULLVALUE === (key = @key_proc.call(e))
+	  return
+	end
+	
+	unless idx = @pvs.find_index{|pv| key < @key_proc.call(pv)}
+	  idx = @pvs.size
+	end
+
+#Log::debug(self, "#{@pvs.inspect}")
+#Log::debug(self, "#{idx}")
+	
+
+	export = @exports[idx]
+	export.push e
+	@counter[idx] += 1
+      end
+
+      def hash_key(e)
+	@key_proc.yield(e)
+      end
+    end
+
+    class PPostSort<PSingleExportFilter
+      Processor.def_export self
+
+      def initialize(id, ntask, bjob, opts, block_source)
+	super
+	@block_source = block_source
+
+	@buffering_policy = @opts[:buffering_policy]
+	@buffering_policy ||= CONF.SORT_BUFFERING_POLICY
+
+	unless CONF.BUG234
+	  @cmp_optimize = CONF.SORT_CMP_OPTIMIZE
+	  @cmp_optimize = opts[:cmp_optimize] if opts.key?(:cmp_optimize)
+	end
+      end
+
+      def basic_each(&block)
+	@key_value_buffer = 
+	  eval("#{@buffering_policy[:buffering_class]}").new(self, @buffering_policy)
+	if @cmp_optimize
+	  @hash_proc = eval("proc{#{@block_source.source}}")
+	else
+	  @hash_proc = BBlock.new(@block_source, @context, self)
+	end
+
+	@input.each do |e|
+	  @key_value_buffer.push(e)
+	  e = nil
+	end
+	@key_value_buffer.each do |values|
+	  values.each(&block)
+	end
+	@key_value_buffer = nil
+      end
+
+      def hash_key(e)
+	@hash_proc.yield(e)
+      end
+    end
+
+
+  end
+end
+
+

data/lib/fairy/node/p-task.rb

@@ -0,0 +1,113 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  class PTask
+    Processor.def_export self
+
+    END_OF_STREAM = :END_OF_STREAM
+
+    ST_INIT = :ST_INIT
+    ST_ACTIVATE = :ST_ACTIVATE
+    ST_FINISH = :ST_FINISH
+
+    def initialize(id, processor)
+      @id = id
+      Log::info self, "CREATE PTask"
+
+      @processor = processor
+
+      @njobs = []
+      @njob_seq = -1
+      @njob_seq_mutex = Mutex.new
+
+      @status = ST_INIT
+      @status_mon = processor.njob_mon
+      @status_cv = @status_mon.new_cv
+
+      start_watch_status
+    end
+
+    attr_reader :processor
+
+    def log_id
+      "PTask[#{@id}]"
+    end
+
+    #
+    # njob methods
+    #
+    def njob_next_id
+      @njob_seq_mutex.synchronize do
+	@njob_seq += 1
+
+	format("%02d-%02d", @id, @njob_seq)
+      end
+    end
+
+    def create_njob(njob_class_name, bjob, opts, *rests)
+      klass = @processor.import(njob_class_name)
+#Log::debugf(self, "KKKKKKKKKKLAS: %s", klass)
+      njob = klass.new(njob_next_id, self, bjob, opts, *rests)
+      @njobs.push njob
+      Log.debugf(self, "Njob number of %d", @njobs.size)
+      njob
+    end
+    DeepConnect.def_method_spec(self, "REF create_njob(VAL, REF, VAL, *DEFAULT)")
+
+    def abort_running
+      @status_mon.synchronize do
+	@njobs.last.abort_running unless [ST_INIT, ST_FINISH].include?(@status)
+      end
+    end
+
+    #
+    # status methods.
+    #
+    def status=(val)
+      @status_mon.synchronize do
+	@status = val
+	@status_cv.broadcast
+      end
+    end
+
+    def start_watch_status
+      # 初期状態通知
+      notice_status(@status)
+
+      @processor.njob_mon.entry do
+	@status_mon.synchronize do
+	  old_status = nil
+	  loop do
+	    @status_cv.wait_while{old_status == @status}
+	    old_status = @status
+	    notice_status(@status)
+	    break if @status == ST_FINISH
+	  end
+	end
+      end
+      nil
+    end
+
+    def update_status(node, st)
+      self.status = st
+    end
+
+    def notice_status(st)
+#      @status_mon.entry do
+	@processor.update_status(self, st)
+#      end
+    end
+
+    def to_s
+      "#<#{self.class}:##{@id} #{@njobs.collect{|f| f.class.to_s}.join('-').gsub(/Fairy::/, "")}>"
+    end
+
+  end
+end
+
+
+
+

data/lib/fairy/node/p-there.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PThere<PFilter
+    Processor.def_export self
+
+    include PSingleExportable
+
+    def initialize(id, ntask, bjob, opts)
+      super
+    end
+
+    def open(nthere_place)
+      self.no = nthere_place.no
+      @enumerable = nthere_place.enumerable
+    end
+
+    def basic_each(&block)
+      @enumerable.each do |e| 
+	if e.__deep_connect_reference? && !PORT_KEEP_IDENTITY_CLASS_SET[e.class]
+	  e = e.deep_connect_deep_copy
+	end
+	block.call  e
+      end
+    end
+
+#     def start
+#       super do
+# 	@enumerable.each do |e| 
+# 	  if e.__deep_connect_reference? && !PORT_KEEP_IDENTITY_CLASS_SET[e.class]
+# 	    e = e.deep_connect_deep_copy
+# 	  end
+# 	  @export.push e
+# 	end
+#       end
+#     end
+  end
+end