fairy 0.6.0

data/lib/fairy/node/p-basic-group-by.rb

@@ -0,0 +1,252 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+
+module Fairy
+  class PBasicGroupBy<PIOFilter
+    Processor.def_export self
+
+    ST_ALL_IMPORTED = :ST_ALL_IMPORTED
+    ST_WAIT_EXPORT_FINISH = :ST_WAIT_EXPORT_FINISH
+    ST_EXPORT_FINISH = :ST_EXPORT_FINISH
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+
+      @exports = {}
+      @exports_queue = Queue.new
+      
+      @counter = {}
+
+      #start_watch_exports
+    end
+
+    def add_export(key, export)
+      @exports[key] = export
+#      @exports_queue.push [key, export]
+      # [BUG#171]同期処理でないとまずい.
+      @bjob.add_exports(key, export, self)
+    end
+
+    def start_export
+      Log::debug(self, "START_EXPORT")
+
+      start do
+	hash_opt = @opts[:grouping_optimize]
+	hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
+	
+	if hash_opt
+	  @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
+	else
+	  @key_proc = BBlock.new(@block_source, @context, self)
+	end
+	
+	policy = @opts[:postqueuing_policy]
+	begin
+	  @input.each do |e|
+	    key = hash_key(e)
+	    if Import::CTLTOKEN_NULLVALUE === key
+	      next
+	    end
+	    export = @exports[key]
+	    unless export
+	      export = Export.new(policy)
+	      export.njob_id = @id
+	      export.add_key(key)
+	      add_export(key, export)
+	      @counter[key] = 0
+	    end
+	    export.push e
+	    @counter[key] += 1
+	  end
+	rescue
+	  Log::debug_exception(self)
+	  raise
+	ensure
+	  @exports_queue.push nil
+	  @exports.each_pair{|key, export| 
+	    Log::debug(self, "G0 #{key} => #{@counter[key]}")	    
+	    export.push END_OF_STREAM}
+	end
+      end
+    end
+
+    def terminate
+      @wait_cv = @terminate_mon.new_cv
+      wait_export_finish
+      super
+    end
+
+    def hash_key(e)
+      @key_proc.yield(e)
+    end
+
+    def wait_export_finish
+
+      Log::debug(self, "G1")
+
+      self.status = ST_ALL_IMPORTED
+
+      Log::debug(self, "G2")
+      # ここの位置が重要
+      self.status = ST_WAIT_EXPORT_FINISH
+      # ここもいまいち
+      Log::debug(self, "G3")
+      @exports.each_pair do |key, export|
+	Log::debug(self, "G3.WAIT #{key}")
+	export.fib_wait_finish(@wait_cv)
+      end
+      Log::debug(self, "G4")
+      self.status = ST_EXPORT_FINISH
+    end
+
+  end
+
+  class PBasicMGroupBy<PIOFilter
+    Processor.def_export self
+
+    ST_ALL_IMPORTED = :ST_ALL_IMPORTED
+    ST_WAIT_EXPORT_FINISH = :ST_WAIT_EXPORT_FINISH
+    ST_EXPORT_FINISH = :ST_EXPORT_FINISH
+
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+#      @key_proc = eval("proc{#{@block_source}}", TOPLEVEL_BINDING)
+#      @key_proc = @context.create_proc(@block_source)
+      @key_proc = BBlock.new(@block_source, @context, self)
+
+      @exports = {}
+      @exports_queue = Queue.new
+
+#      start_watch_exports
+    end
+
+    def add_export(key, export)
+      @exports[key] = export
+#      @exports_queue.push [key, export]
+      # [BUG#171]同期処理でないとまずい.
+      @bjob.add_exports(key, export, self)
+    end
+
+    def start_export
+      Log::debug(self, "START_EXPORT")
+
+      start do
+	hash_opt = @opts[:grouping_optimize]
+	hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil?
+
+	if hash_opt
+	  @key_proc = eval("proc{#{@block_source.source}}", @context.binding)
+	else
+	  @key_proc = BBlock.new(@block_source, @context, self)
+	end
+
+	policy = @opts[:postqueuing_policy]
+        begin
+          @input.each do |e|
+            keys = @key_proc.yield(e)
+            keys = [keys] unless keys.kind_of?(Array)
+            
+            for key in keys 
+	      if Import::CTLTOKEN_NULLVALUE === key
+		next
+	      end
+              export = @exports[key]
+              unless export
+                export = Export.new(policy)
+		export.njob_id = @id
+                export.add_key(key)
+                add_export(key, export)
+              end
+              export.push e
+            end
+          end
+        rescue
+	  Log::debug_exception(self)
+	  raise
+        ensure
+          @exports_queue.push nil
+          @exports.each_pair do |key, export| 
+	    next unless export
+	    export.push END_OF_STREAM
+	  end
+        end
+      end
+    end
+
+    def terminate
+      @wait_cv = @terminate_mon.new_cv
+      wait_export_finish
+      super
+    end
+
+
+#     def start
+#       super do
+# 	policy = @opts[:postqueuing_policy]
+# 	@import.each do |e|
+# 	  keys = @key_proc.yield(e)
+# 	  keys = [keys] unless keys.kind_of?(Array)
+	  
+# 	  for key in keys 
+# 	    export = @exports[key]
+# 	    unless export
+# 	      export = Export.new(policy)
+# 	      export.add_key(key)
+# 	      add_export(key, export)
+# 	    end
+# 	    export.push e
+# 	  end
+# 	end
+# 	@exports.each{|key, export| export.push END_OF_STREAM}
+# 	wait_export_finish
+#       end
+#     end
+
+    def wait_export_finish
+
+Log::debug(self, "G1")
+      self.status = ST_ALL_IMPORTED
+
+Log::debug(self, "G2")
+      # すべての, exportのoutputが設定されるまで待っている
+      # かなりイマイチ
+#      for key, export in @exports
+#	export.output
+#      end
+
+      # ここの位置が重要
+      self.status = ST_WAIT_EXPORT_FINISH
+      # ここもいまいち
+Log::debug(self, "G3")
+      @exports.each_pair do |key, export|
+	next unless export
+Log::debug(self, "G4.WAIT #{key}")
+	export.fib_wait_finish(@wait_cv)
+      end
+Log::debug(self, "G5")
+      self.status = ST_EXPORT_FINISH
+    end
+
+#     def start_watch_exports
+#       Thread.start do
+# 	loop do
+# 	  key, export = @exports_queue.pop
+# 	  notice_exports(key, export)
+# 	end
+#       end
+#       nil
+#     end
+
+#     def notice_exports(key, export)
+#       @bjob.update_exports(key, export, self)
+#     end
+  end
+end
+
+

data/lib/fairy/node/p-direct-product.rb

@@ -0,0 +1,153 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  module PDirectProduct
+    class PPreFilter<PIOFilter
+      Processor.def_export self
+
+      def initialize(id, ntask, bjob, opts, block_source)
+	super
+	@block_source = block_source
+
+	@exports = nil
+	@exports_mutex = Mutex.new
+	@exports_cv = ConditionVariable.new
+      end
+
+      def input=(input)
+	super
+	start_watch_exports
+      end	
+
+#       def number_of_exports=(n)
+# 	@exports_mutex.synchronize do
+# 	  @exports = []
+# 	  n.times{@exports.push Export.new}
+# 	end
+# 	@exports_cv.broadcast
+#       end
+
+      def start_watch_exports
+	Thread.start do
+	  n = @bjob.number_of_exports
+	  @exports_mutex.synchronize do
+	    @exports = []
+	    n.times do
+
+# 後で検討する
+#	      policy = @opts[:postqueuing_policy]
+#	      exp = Export.new(policy)
+	      exp = Export.new
+	      exp.njob_id = @id
+	      exp.no = @input.no
+	      exp.add_key(@input.key)
+	      exp.output_no_import = 1
+	      @exports.push exp
+	    end
+	  end
+	  @exports_cv.broadcast
+	end
+      end
+
+      def exports
+	@exports_mutex.synchronize do
+	  while @exports.nil?
+	    @exports_cv.wait(@exports_mutex)
+	  end
+	end
+	@exports
+      end
+      
+      def start_export
+	start do
+	  begin
+	    @input.each do |e|
+	      exports.each{|exp| exp.push e}
+	    end
+	  ensure
+	    exports.each{|exp| exp.push :END_OF_STREAM}
+	  end
+ 	end
+       end
+    end
+
+    class PPostFilter<PSingleExportFilter
+      Processor.def_export self
+
+      def initialize(id, processor, bjob, opts, block_source)
+	super
+	@block_source = block_source
+	
+	@other_imports = nil
+	@other_imports_mutex = Mutex.new
+	@other_imports_cv = ConditionVariable.new
+      end
+
+      def input=(input)
+	@input = input
+#	self.no = input.no
+	self.key = input.key
+      end
+
+      def other_inputs=(exports)
+	@other_imports_mutex.synchronize do
+	  @other_imports = exports.collect{|exp|
+# 後で検討する
+#           policy = @opts[:prequeuing_policy]
+#	    imp = Import.new(policy)
+	    imp = Import.new
+	    imp.no = exp.no
+	    imp.add_key(exp.key)
+	    imp.no_import = 1
+	    imp.set_log_callback do |n, key| 
+	      Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	    end
+
+	    exp.output = imp
+	    imp
+	  }
+	  @other_imports_cv.broadcast
+	end
+      end
+
+      def other_imports
+	@other_imports_mutex.synchronize do
+	  while !@other_imports
+	    @other_imports_cv.wait(@other_imports_mutex)
+	  end
+	  @other_imports
+	end
+      end
+
+      def basic_each(&block)
+	@map_proc = BBlock.new(@block_source, @context, self)
+
+	elements = []
+	elements.push @input.to_a
+	elements.push *other_imports.collect{|i| i.to_a}
+
+	idxs = elements.collect{|e| 0}
+	max_idxs = elements.collect{|e| e.size}
+
+	cont = true
+	while cont
+	  e = elements.zip(idxs).collect{|ary, idx| ary[idx]}
+	  block.call @map_proc.yield *e
+
+	  (idxs.size-1).downto(0) do |idx|
+	    idxs[idx] += 1
+	    break if idxs[idx] < max_idxs[idx]
+	    idxs[idx] = 0
+	    cont = false if idx == 0 && idxs[idx] == 0
+	  end
+	end
+      end
+    end
+  end
+end

data/lib/fairy/node/p-exec.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PExec<PSingleExportInput
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts=nil)
+      super
+    end
+
+    def start(nfileplace)
+      @nfileplace = nfileplace
+      self.no = nfileplace.no
+      self
+    end
+    alias open start
+    DeepConnect::def_method_spec(self, "REF start(VAL)")
+    DeepConnect::def_method_spec(self, "REF open(VAL)")
+
+    def basic_each(&block)
+      block.call @nfileplace.url
+    end
+  end
+end

data/lib/fairy/node/p-filter.rb

@@ -0,0 +1,363 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "thread"
+
+require "fairy/node/p-task"
+
+require "fairy/share/block-source"
+require "fairy/share/port"
+
+module Fairy
+
+  class PFilter
+    Processor.def_export self
+
+    END_OF_STREAM = :END_OF_STREAM
+
+    ST_INIT = :ST_INIT
+    ST_ACTIVATE = :ST_ACTIVATE
+    ST_FINISH = :ST_FINISH
+
+    DeepConnect.def_single_method_spec(self, "REF new(DEFAULT, REF, REF, VAL, *DEFAULT)")
+
+    def initialize(id, ntask, bjob, opts={}, *rests)
+      @id = id
+      @log_id = format("%s[%s]", self.class.name.sub(/Fairy::/, ''), @id)
+
+      Log::info self, "CREATE NJOB: #{self.class}"
+      @ntask = ntask
+      @bjob = bjob
+      @opts = opts
+
+      @IGNORE_EXCEPTION = CONF.IGNORE_EXCEPTION_ON_FILTER
+      @IGNORE_EXCEPTION = @opts[:ignore_exception] if @opts.key?(:ignore_exception)
+
+      @main_thread = nil
+
+      @context = Context.new(self)
+#       @begin_block = nil
+#       if @opts[:BEGIN]
+# 	@begin_block = BBlock.new(@opts[:BEGIN], @context, self)
+#       end
+#       @end_block = nil
+#       if @opts[:END]
+# 	@end_block = BBlock.new(@opts[:END], @context, self)
+#       end
+
+      @begin_block_source = nil
+      if @opts[:BEGIN]
+	@begin_block_source = @opts[:BEGIN]
+	@begin_block_exec_p = false
+      end
+      @end_block_source = nil
+      if @opts[:END]
+	@end_block_source = @opts[:END]
+      end
+
+      @no = nil
+      @no_mutex = Mutex.new
+      @no_cv = ConditionVariable.new
+
+      @key = nil
+      @key_mutex = Mutex.new
+      @key_cv = ConditionVariable.new
+
+      @status = ST_INIT
+      @status_mon = processor.njob_mon
+      @status_cv = @status_mon.new_cv
+      notice_status(@status)
+
+      @terminate_mon = processor.njob_mon
+
+      @in_each = nil
+      @in_each_mutex = Mutex.new
+
+#      start_watch_status
+    end
+
+    attr_reader :id
+    attr_reader :log_id
+
+    attr_reader :IGNORE_EXCEPTION
+
+    attr_reader :ntask
+    def processor
+      @ntask.processor
+    end
+    
+    def no
+#Log::debug(self, "XXXXXXXXXXXXXXXXXXXXXXXX")
+#Log::debug_backtrace
+      @no_mutex.synchronize do
+	while !@no
+	  @no_cv.wait(@no_mutex)
+	end
+	@no
+      end
+    end
+
+    def no=(no)
+      @no_mutex.synchronize do
+	@no = no
+	@no_cv.broadcast
+	@no
+      end
+    end
+
+    def key
+#       @key_mutex.synchronize do
+# 	while !@key
+# 	  @key_cv.wait(@key_mutex)
+# 	end
+# 	@key
+#       end
+      @key
+    end
+
+    def key=(key)
+#       @key_mutex.synchronize do
+# 	@key = key
+# 	@key_cv.broadcast
+# 	@key
+#       end
+      @key=key
+    end
+
+    def start_export
+      ERR::Raise ERR::INTERNAL::ShouldDefineSubclass
+    end
+
+    def start(&block)
+      Log::info self, "START PROCESSING: #{self.class}"
+
+      start_watch_status
+
+      @main_thread = Thread.start{
+	begin
+	  self.status = ST_ACTIVATE
+	  if @begin_block_source
+	    bsource = BSource.new(@begin_block_source, @context, self)
+	    bsource.evaluate
+	  end
+	  begin
+	    basic_start &block
+	  ensure
+	    if @end_block_source
+	      bsource = BSource.new(@end_block_source, @context, self)
+	      bsource.evaluate
+	    end
+
+	    @main_thread = nil
+	    @terminate_mon.entry terminate_proc
+	    Log::info self, "FINISH PROCESSING: #{self.class}"
+	  end
+	rescue Exception
+	  Log::error_exception(self)
+	  handle_exception($!)
+	  raise
+	end
+      }
+      nil
+    end
+
+    def terminate_proc
+      proc{|*dummy| terminate}
+    end
+
+    def terminate
+      self.status = ST_FINISH
+    end
+
+    def basic_start(&block)
+      block.call
+    end
+
+    def each(&block)
+      begin
+	@in_each = true
+
+	if @begin_block_source
+	  bsource = BSource.new(@begin_block_source, @context, self)
+	  bsource.evaluate
+	  @begin_block_exec_p = true
+	end
+	begin
+	  basic_each do |e|
+	    case e
+	    when Import::CTLTOKEN_NULLVALUE
+	      next
+	    else
+	      begin
+		block.call e
+	      rescue
+		if @IGNORE_EXCEPTION
+		  Log::warn(self, "IGNORE_EXCEPTON!!")
+		  Log::warn(self, "Entity: #{e.inspect}")
+		  Log::error_exception(self)
+		  next
+		else
+		  raise
+		end
+	      end
+	    end
+	  end
+	ensure
+	  if @end_block_source
+	    bsource = BSource.new(@end_block_source, @context, self)
+	    bsource.evaluate
+	  end
+	end
+      rescue @context.class::GlobalBreakFromOther
+	Log::debug(self, "CAUGHT GlobalBreak From Other")
+	global_break_from_other
+	
+      rescue LocalJumpError, @context.class::GlobalBreak
+	Log::debug(self, "CAUGHT GlobalBreak")
+	global_break
+
+      rescue Exception
+	Log::error_exception(self)
+	handle_exception($!)
+	raise
+      ensure
+	@in_each = false
+      end
+      nil
+    end
+
+    def next
+      if @begin_block_source && @begin_block_exec_p
+	bsource = BSource.new(@begin_block_source, @context, self)
+	bsource.evaluate
+	@begin_block_exec_p = true
+      end
+      begin
+	ret = basic_next
+      ensure
+	if ret == :END_OF_STREAM
+	  if @end_block_source
+	    bsource = BSource.new(@end_block_source, @context, self)
+	    bsource.evaluate
+	  end
+	end
+      end
+    end
+
+    def global_break
+      Thread.start{@bjob.break_running(self)}
+#      Thread.current.exit
+      self.status = ST_FINISH
+      # 他のスレッドはとめていない
+    end
+
+    def global_break_from_other
+      self.status = ST_FINISH
+    end
+
+    def break_running
+      if @in_each
+	@main_thread.raise @context.class::GlobalBreakFromOther
+#      @main_thread.exit if @main_thread
+	self.status = ST_FINISH
+      end
+      # 他のスレッドはとめていない
+    end
+
+    def abort_running
+      @main_thread.exit if @main_thread
+    end
+
+    def status=(val)
+#Log::debugf(self, "STATUS_CHANGE: %s", val)
+      @status_mon.synchronize do
+	@status = val
+	@status_cv.broadcast
+      end
+    end
+
+    def start_watch_status
+      # 初期状態通知
+      notice_status(@status)
+
+      @status_mon.entry do
+	@status_mon.synchronize do
+	  old_status = nil
+	  loop do
+	    @status_cv.wait_while{old_status == @status}
+	    old_status = @status
+	    notice_status(@status)
+	    break if @status == ST_FINISH
+	  end
+	end
+      end
+      nil
+    end
+
+    def notice_status(st)
+      @status_mon.entry do
+	@bjob.update_status(self, st)
+	@ntask.update_status(self, st)
+      end
+    end
+
+#     # block create
+#     def create_block(source)
+#       unless Fairy.const_defined?(:Pool)
+# 	pool_dict = @bjob.pool_dict
+# 	Fairy.const_set(:Pool, pool_dict)
+#       end
+#       eval("def Pool = Fairy::Pool", TOPLEVEL_BINDING)
+
+#       binding = eval("def fairy_binding; binding; end; fairy_binding",
+# 		      TOPLEVEL_BINDING, 
+# 		      __FILE__,
+# 		      __LINE__ - 3)
+#     end
+
+    def handle_exception(exp)
+      @bjob.handle_exception(exp)
+    end
+
+    class Context
+      def initialize(njob)
+	@Pool = njob.instance_eval{@bjob.pool_dict}
+#Log::debug(self, @Pool.peer_inspect)
+	@JobPool = njob.instance_eval{@bjob.job_pool_dict}
+	#      @Import = njob.instance_eval{@import}
+	#      @Export = njob.instance_eval{@export}
+	@__context = context
+
+	@IGNORE_EXCEPTION = njob.IGNORE_EXCEPTION
+
+#      Log::debugf(self, "CONTEXT: %s", eval("@Pool", self.binding))
+      end
+
+      attr_reader :IGNORE_EXCEPTION
+
+      def context
+	__binding
+      end
+
+      class GlobalBreak<Exception;end
+      class GlobalBreakFromOther<Exception;end
+#      class LocalBreak<Exception;end
+      def global_break
+	Thread.current.raise GlobalBreak
+      end
+      alias gbreak global_break
+
+#       def local_break
+# 	Thread.current.raise LocalBreak
+#       end
+
+      alias __binding binding
+      def binding
+	@__context
+      end
+      alias bind binding
+    end
+  end
+end