fairy 0.6.0

data/lib/fairy/logger.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  class Logger
+
+    LOG_FILE = CONF.LOG_FILE
+
+    FLUSH_INTERVAL = CONF.LOG_FLUSH_INTERVAL
+    MARK_INTERVAL = CONF.LOG_MARK_INTERVAL
+    LOG_ROTATE_INTERVAL = CONF.LOG_ROTATE_INTERVAL
+
+    def initialize(path = LOG_FILE)
+      @log_file_path = path
+
+      open_log
+      
+      @mutex = Mutex.new
+      @buffered = false
+      
+      start_flusher
+    end
+
+    def open_log
+      begin
+	@log_out = File.open(@log_file_path, "a+")
+      rescue Errno::ENOENT
+	ERR::Fail ERR::NoLogDir, @log_file_path
+      rescue
+	raise
+      end
+      @log_open_time = Time.now
+      @log_out.puts @log_open_time.strftime("%Y/%m/%d %H:%M:%S -- LOGGER START --")
+      @log_out.flush
+
+      @marked_time = @log_open_time
+    end
+
+    def start_flusher
+      Thread.start do
+	loop do
+	  sleep FLUSH_INTERVAL
+	  @mutex.synchronize do
+	    if @buffered
+	      @log_out.flush
+	      @buffered = false
+	    end
+	    now = Time.now 
+	    if MARK_INTERVAL && now - @marked_time > MARK_INTERVAL
+	      @log_out.puts now.strftime("%Y/%m/%d %H:%M:%S -- MARK --")
+	      @log_out.flush
+	      @marked_time = now
+	    end
+	    if LOG_ROTATE_INTERVAL && now - @log_open_time > LOG_ROTATE_INTERVAL
+	      log_rotate
+	    end
+	  end
+	end
+      end
+    end
+
+    def message(message)
+      @mutex.synchronize do
+	@log_out.puts message
+	@buffered = true
+      end
+      nil
+    end
+
+    def messages(messages)
+      @mutex.synchronize do
+	messages.each do |m|
+	  @log_out.puts m
+	end
+	@buffered = true
+      end
+      nil
+    end
+    DeepConnect.def_method_spec(self, "REF messages(DVAL)")
+
+    def log_rotate
+      @log_out.close
+      log_back = "#{@log_file_path}.BAK-$$"
+      File.rename(@log_file_path, log_back)
+      open_log
+
+      Thread.start do
+	files = Dir.glob("#{@log_file_path}.[0-9]*").sort{|f1, f2| f2 <=> f1}
+	while files.size  >= CONF.LOG_ROTATE_N
+	  File.unlink(files.shift)
+	end
+
+	files.each do |f|
+	  fn = f.sub(/\.[0-9]+/){|n| n.succ}
+	  File.rename(f, fn)
+	  if /.*\.gz$/ !~ fn
+	    system("gzip #{fn}")
+	  end
+	end
+
+	File.rename(log_back, "#{@log_file_path}.0")
+      end
+    end
+  end
+end

data/lib/fairy/master/addins.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+bjob_dir = File.dirname(__FILE__)
+subdir = File.basename(File.dirname(bjob_dir))
+bjob_name = File.basename(bjob_dir)
+for bjob in Dir.glob("#{bjob_dir}/*.rb")
+  base = File.basename(bjob)
+  case base
+  when /18.rb$/
+    next if RUBY_VERSION >= "1.9.0"
+  when /19.rb$/
+    next unless RUBY_VERSION >= "1.9.0"
+  end
+  require [subdir, bjob_name, base].join("/")
+end
+
+

data/lib/fairy/master/atom.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+module Fairy
+
+  class Atom
+    def initialize(receiver, message, *args)
+      @receiver = receiver
+      @message = message
+      @args = args
+    end
+
+    attr_reader :receiver, :message, :args
+    
+  end
+end
+
+  

data/lib/fairy/master/c-barrier.rb

@@ -0,0 +1,283 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "forwardable"
+
+require "deep-connect"
+
+require "fairy/master/c-filter"
+require "fairy/master/c-inputtable"
+require "fairy/master/c-io-filter"
+
+require "fairy/share/block-source"
+
+module Fairy
+  class CBarrier<CIOFilter
+    extend Forwardable
+
+    Controller.def_export self
+
+#    include BInputtable
+
+    DeepConnect.def_single_method_spec(self, "REF new(REF, VAL)")
+
+    def initialize(controller, opts)
+      super
+      for k, val in opts.dup
+	case k
+	when :mode
+	  @mode = CBarrierMode.create(self, val, opts)
+	when :cond
+	  @cond = CBarrierCond.create(self, val, opts)
+	when :buffer
+	  @buffer = CBarrierBuffer.create(self, val, opts)
+	else
+	end
+      end
+    end
+
+    def each_assigned_filter(&block)
+      @mode.wait_exportable
+      @buffer.each_assigned_filter(&block)
+    end
+
+    def_delegator :@mode, :wait_export
+
+    def_delegator :@cond, :wait_cond
+
+    def_delegator :@buffer, :input=
+    def_delegator :@buffer, :output=
+
+    def_delegator :@buffer, :node_arrived?
+    def_delegator :@buffer, :data_arrived?
+    def_delegator :@buffer, :all_data_imported?
+
+    #
+    module Factory
+      def self.extended(mod)
+	mod.init_fact
+      end
+
+      def init_fact
+	@ModeName2Class = {}
+      end
+      
+      def create(bbarrier, mode, *opts)
+	klass = @ModeName2Class[mode]
+	ERR::Raise ERR::NoSuchMode, mode unless klass
+	
+	mode = klass.new(bbarrier, mode, *opts)
+	mode
+      end
+
+      def register_mode(mode, klass)
+	@ModeName2Class[mode] = klass
+      end
+    end
+
+    module Mode
+      def initialize(bbarrier, mode, *opts)
+	@bbarrier = bbarrier
+	@mode = mode
+	@opts = opts.last
+#	Log::debug self, self.class.superclass.name
+	begin
+	  super(*opts)
+	rescue
+	  # for ruby 1.9.1
+	  super()
+	end
+      end
+    end
+    
+    #
+    class CBarrierMode
+      extend Factory
+      include Mode
+
+      def initialize(bbarrier, mode, *opts)
+	super
+	@opts = opts
+      end
+
+    end
+
+    class CBarrierNodeCreationMode<CBarrierMode
+      CBarrierMode.register_mode(:NODE_CREATION, self)
+
+      def wait_exportable
+	@bbarrier.wait_cond
+      end
+
+      def wait_export
+	true
+      end
+
+    end
+
+    class CBarrierStreamMode<CBarrierMode
+      CBarrierMode.register_mode(:STREAM, self)
+
+      def wait_exportable
+	true
+      end
+
+      def wait_export
+	@bbarrier.wait_cond
+      end
+
+    end
+
+    #
+    class CBarrierCond
+      extend Factory
+      include Mode
+      
+      def self.create(bbarrier, mode, opts=nil)
+	if mode.kind_of?(BlockSource)
+	  opts[:BLOCK_SOURCE] = mode
+	  super(bbarrier, :BLOCK_COND, opts)
+	else
+	  super(bbarrier, mode, opts)
+	end
+      end
+
+      def wait_cond
+	ERR::Raise ERR::NoImpliment, "wait_cond"
+      end
+
+    end
+
+    class CBarrierNodeArrivedCond<CBarrierCond
+      CBarrierCond.register_mode(:NODE_ARRIVED, self)
+
+      def wait_cond
+	@bbarrier.node_arrived?
+      end
+
+    end
+
+    class CBarrierDataArrivedCond<CBarrierCond
+      CBarrierCond.register_mode(:DATA_ARRIVED, self)
+
+      def wait_cond
+	@bbarrier.data_arrived?
+      end
+
+    end
+
+    class CBarrierAllDataCond<CBarrierCond
+      CBarrierCond.register_mode(:ALL_DATA, self)
+
+      def wait_cond
+	@bbarrier.all_data_imported?
+      end
+    end
+
+    class CBarrierBlockCond<CBarrierCond
+      CBarrierCond.register_mode(:BLOCK_COND, self)
+
+      def initialize(bbarrier, mode, opts)
+	super(bbarrier, mode, opts)
+
+	if @opts[:BEGIN]
+	  bs = BScript.new(@opts[:BEGIN], 
+			   @bbarrier.instance_eval{@context}, 
+			   @bbarrier)
+	  bs.evaluate
+	end
+	@block_source = @opts[:BLOCK_SOURCE]
+	@block = BBlock.new(@block_source, 
+			    @bbarrier.instance_eval{@context}, 
+			    @bbarrier)
+	# @opts[:END] は未サポート
+      end
+
+      def wait_cond
+	@block.call
+      end
+    end
+
+    #
+    class CBarrierBuffer<CIOFilter
+      extend Factory
+      include Mode
+    end
+
+    class CBarrierMemoryBuffer<CBarrierBuffer
+      CBarrierBuffer.register_mode(:MEMORY, self)
+
+      def initialize(bbarrier, mode, opts=nil)
+	super(bbarrier, mode, bbarrier.instance_eval{@controller}, opts)
+      end
+
+      def node_arrived?
+	number_of_nodes
+      end
+
+      def data_arrived?
+	@nodes_status_mutex.synchronize do
+	  while !all_node_data_arrived?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+	true
+      end
+
+      def all_data_imported?
+	@nodes_status_mutex.synchronize do
+	  while !all_node_data_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+	true
+      end
+
+      def node_class_name
+	"PBarrierMemoryBuffer"
+      end
+
+      def wait_export
+	@bbarrier.wait_export
+      end
+
+      def all_node_data_arrived?
+	return false unless @number_of_nodes
+
+	data_arrived = true
+	each_node(:exist_only) do |node|
+	  st = @nodes_status[node]
+	  data_arrived &&= [:ST_ACTIVATE, :ST_ALL_IMPORTED, :ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH].include?(st)
+	end
+	data_arrived
+      end
+
+      def all_node_data_imported?
+	return false unless @number_of_nodes
+
+	all_data_imported = true
+	each_node(:exist_only) do |node|
+	  st = @nodes_status[node]
+	  s = [:ST_ALL_IMPORTED, :ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH].include?(st)
+	  all_data_imported &&= [:ST_ALL_IMPORTED, :ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH].include?(st)
+	end
+	all_data_imported
+      end
+
+    end
+
+    class CBarrierFileBuffer<CBarrierBuffer
+      CBarrierBuffer.register_mode(:FILE, self)
+
+      def node_class_name
+	"PBarrier::PBarrierFileBuffer"
+      end
+    end
+
+  end
+end
+
+
+

data/lib/fairy/master/c-basic-group-by.rb

@@ -0,0 +1,250 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/master/c-io-filter"
+require "fairy/master/c-inputtable"
+
+module Fairy
+  class CBasicGroupBy<CIOFilter
+    Controller.def_export self
+
+    def initialize(controller, opts, block_source)
+      super
+      @block_source = block_source
+
+      @no_of_exports = 0
+
+      # key -> [export, ...]
+      @exports = {}
+      @exports_mutex = Mutex.new
+      @exports_cv = ConditionVariable.new
+
+#      @pre_exports_queue = Queue.new
+      @exports_queue = Queue.new
+
+      @each_export_by_thread = nil
+      @each_export_by_thread_mutex = Mutex.new
+    end
+
+    def start_create_nodes
+      super
+
+      start_watch_all_node_imported
+    end
+
+#    def each_export(&block)
+#      while pair = @exports_queue.pop
+#	block.call pair
+#      end
+#    end
+
+#     def next_filter(mapper)
+#       ret = super
+#       unless ret
+# 	@each_export_by_thread_mutex.synchronize do
+# 	  @each_export_by_thread.join if @each_export_by_thread
+# 	end
+#       end
+#       ret 
+#     end
+
+    def each_assigned_filter(&block)
+      super
+
+      @each_export_by_thread_mutex.synchronize do
+	@each_export_by_thread.join if @each_export_by_thread
+      end
+    end
+
+#     def each_export_by(njob, mapper, &block)
+#       return if @each_export_by_thread
+
+#       begin
+# 	while pair = @exports_queue.pop
+# 	  exp, njob = pair
+# 	  Log::debug(self, "EXPORT_BY, #{exp.key}")
+# 	  block.call exp
+# 	end
+#       rescue
+# 	Log::fatal_exception
+#       end
+#       @each_export_by_thread = true
+#     end
+
+    def each_export_by(njob, mapper, &block)
+      @each_export_by_thread_mutex.synchronize do
+	return if @each_export_by_thread
+
+	@each_export_by_thread = Thread.start{
+	  # すべての njob がそろうまで待つ
+	  # 後段が先にスケジュールされてデッドロックするのを避けるため.
+	  number_of_nodes
+
+	  begin
+	    while pair = @exports_queue.pop
+	      exp, njob = pair
+Log::debug(self, "EXPORT_BY, #{exp.key}")
+	      block.call exp
+
+	      @exports_mutex.synchronize do
+		if @exports[exp.key].first == exp
+		  @exports[exp.key][1..-1].each do |e|
+		    e.output = exp.output
+		  end
+		end
+	      end
+	    end
+	  rescue
+	    Log::fatal_exception
+	    raise
+	  end
+	}
+      end
+    end
+
+    def bind_export(exp, imp)
+      # do nothing
+    end
+
+    def add_exports(key, export, njob)
+      @exports_mutex.synchronize do
+	if exports = @exports[key]
+	  export.output = exports.first.output if exports.first.output?
+	  export.no = exports.first.no
+	  exports.push export
+	else
+	  export.no = @no_of_exports
+	  @no_of_exports += 1
+	  @exports[key] = [export]
+	  @exports_queue.push [export, njob]
+#	  @pre_exports_queue.push [export, njob]
+	end
+      end
+    end
+
+    def update_exports(key, export, njob)
+      add_exports(key, export, njob)
+      nil
+    end
+
+    def node_class_name
+      "PBasicGroupBy"
+    end
+
+    def njob_creation_params
+      [@block_source]
+    end
+
+    def start_watch_all_node_imported
+      Thread.start do
+	# すべての njob がそろうまで待つ
+	# 後段が先にスケジュールされてデッドロックするのを避けるため.
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: S")
+	number_of_nodes
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1")
+
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 2")
+	# すべての exports がそろうまで待つ
+	@nodes_status_mutex.synchronize do
+	  while !all_node_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+	@exports_queue.push nil
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 4")
+	for key, exports in @exports
+	  exports.first.output_no_import = exports.size
+	end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: E")
+      end
+      nil
+    end
+
+    def start_watch_all_node_imported_ORG
+      Thread.start do
+	# すべての njob がそろうまで待つ
+	# 後段が先にスケジュールされてデッドロックするのを避けるため.
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: S")
+	number_of_nodes
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1")
+	# すでに存在するexportsを下流に送る
+	@exports_mutex.synchronize do
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.1")
+	  @pre_exports_queue.push nil
+	  while pair = @pre_exports_queue.pop
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.2: EXP.NO: #{pair[0].no}")
+	    @exports_queue.push pair
+	  end
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 1.3")
+	end
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 2")
+	# すべての exports がそろうまで待つ
+	@nodes_status_mutex.synchronize do
+	  while !all_node_imported?
+	    @nodes_status_cv.wait(@nodes_status_mutex)
+	  end
+	end
+
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 3")
+	# 残りのexportsを下流に送る
+	@pre_exports_queue.push nil
+	while pair = @pre_exports_queue.pop
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 3.1: EXP.NO: #{pair[0].no}")
+	  @exports_queue.push pair
+	end
+	@exports_queue.push nil
+	
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: 4")
+#Log::debug(self, "START: setting for EXPOTRS.SIZE")
+	for key, exports in @exports
+#	  exports[1..-1].each do |exp|
+#	    exp.output=exports.first.output
+#	  end
+
+#Log::debug(self, "EXPOTRS.SIZE=#{exports.size}")
+	  exports.first.output_no_import = exports.size
+	end
+#Log::debug(self, "END: setting for EXPOTRS.SIZE")
+Log::debug(self, "START_WATCH_ALL_NODE_IMPORTED: E")
+      end
+      nil
+    end
+
+    def all_node_arrived?
+      @nodes_mutex.synchronize{@number_of_nodes}
+    end
+
+    def all_node_imported?
+      # すべてのnjobがそろったか?
+      return false unless @nodes_mutex.synchronize{@number_of_nodes}
+
+      each_node(:exist_only) do |node|
+	st = @nodes_status[node]
+	# こちらはNG: outputが設定されていないとまずい.
+	# すべてのnodeがそろったとしてもすべてのexportがそろっているとは限らない
+#	unless [:ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH, :ST_ALL_IMPORTED].include?(st)
+	unless [:ST_FINISH, :ST_EXPORT_FINISH, :ST_WAIT_EXPORT_FINISH].include?(st)
+	  return false
+	end
+      end
+      true
+    end
+  end
+
+  class CBasicMGroupBy<CBasicGroupBy
+    Controller.def_export self
+
+    def node_class_name
+      "PBasicMGroupBy"
+    end
+  end
+
+
+end