fairy 0.6.0

data/lib/fairy/version.rb

@@ -0,0 +1,8 @@
+#
+# Don't modified this file.
+# This file is auto generation. 
+#
+module Fairy
+    Version = "0.6.0-001"
+end
+

data/lib/fairy.rb

@@ -0,0 +1,206 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "thread"
+
+require "deep-connect"
+
+require "fairy/version"
+require "fairy/share/conf"
+require "fairy/share/log"
+require "fairy/share/locale"
+require "fairy/share/encoding"
+
+module Fairy
+  Conf.configure_common_conf
+
+  @USER_LEVEL_FILTERS = {}
+
+  def Fairy::def_filter(name, opts={}, &definition)
+    name = name.intern if name.kind_of?(String)
+    @USER_LEVEL_FILTERS[name] = definition
+
+    interface_mod = Module.new
+
+    if !opts[:sub]
+      interface_mod.module_eval %{
+        def #{name}(*args)
+	  p = ::Fairy::user_level_filter(:#{name})
+	  ::Fairy::ERR.Raise ::Fairy::ERR::INTERNAL::NoSuchDefiledUserLevelFilter, name unless p
+	  p.call(@fairy, self, *args)			     
+        end
+      }
+    else
+      interface_mod.module_eval %{
+        def #{name}(*args)
+	  p = ::Fairy::user_level_filter(:#{name})
+	  ::Fairy::ERR.Raise ::Fairy::ERR::INTERNAL::NoSuchDefiledUserLevelFilter, name unless p
+  	  sub{|subf, input| p.call(subf, input, *args)}			     
+        end
+      }
+    end
+  Fairy.def_filter_interface interface_mod
+  end
+
+  def Fairy::user_level_filter(name)
+    @USER_LEVEL_FILTERS[name]
+  end
+
+  class Fairy
+
+    def self.create_subfairy(fairy)
+      subfairy = Fairy.allocate
+      subfairy.initialize_subfairy(fairy)
+      subfairy
+    end
+
+    def initialize(master_host = CONF.MASTER_HOST, 
+		   master_port = CONF.MASTER_PORT,
+		   opts = {})
+
+      if master_host.kind_of?(Hash)
+	opts = master_host
+	master_host = CONF.MASTER_HOST
+	master_port = CONF.MASTER_PORT
+      end
+
+      ::Fairy::REPLACE_CONF(Conf.new(CONF, opts))
+
+      Thread.abort_on_exception = CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION
+
+      @name2backend_class = {}
+
+      @deep_connect = DeepConnect.start(0)
+      @master_deepspace = @deep_connect.open_deepspace(master_host, master_port)
+      @master = @master_deepspace.import("Master")
+
+      @controller = @master.assgin_controller
+      @controller.connect(self, CONF)
+
+      @logger = @master.logger
+      Log.type = "[c]"
+      Log.pid = @controller.id
+      Log.logger = @logger
+      Log.set_local_output_dev
+      Log::info self, "fairy connected!!"
+      Log::info self, "\tfairy version: #{Version}"
+      Log::info(self, "\t[Powered By #{RUBY_DESCRIPTION}]") 
+
+      @stdout_mutex = Mutex.new
+
+      if CONF.DEBUG_MONITOR_ON
+	Log::info self, "MONITOR NODE: ON"
+	require "fairy/share/debug"
+	Debug::njob_status_monitor_on(self)
+      end
+    end
+
+    def initialize_subfairy(fairy)
+      @name2backend_class = {}
+      @deep_connect = fairy.instance_eval{@deep_connect}
+      @master_deepspace = fairy.instance_eval{@master_deepspace}
+      @master = fairy.instance_eval{@master}
+
+      @controller = @master.assgin_controller
+      @controller.connect(self, CONF)
+      
+      # Logは親と共有される
+      # なので, IDは親と同じになる(process idなので当たり前)
+      
+      @stdout_mutex = fairy.instance_eval{@stdout_mutex}
+    end
+
+    attr_reader :controller
+
+    def abort
+      @master.terminate_controller(@controller)
+    end
+
+    def name2backend_class(backend_class_name)
+      if klass = @name2backend_class[backend_class_name]
+	return klass 
+      end
+      
+      if klass =  @controller.import(backend_class_name)
+	@name2backend_class[backend_class_name] = klass
+      end
+      klass
+    end
+
+    # pool variables
+    def def_pool_variable(vname, value = nil)
+      @controller.def_pool_variable(vname, value)
+    end
+
+    def pool_variable(vname, *value)
+      @controller.pool_variable(vname, *value)
+    end
+
+    # exception handling
+    def handle_exception(exp)
+      local_exp = nil
+      Log::debug(self, "exception raised: #{exp.class}")
+      Log::debug_exception(self, exp)
+      begin
+	local_exp = exp.dc_deep_copy
+      rescue Exception
+	Thread.main.raise exp
+	raise exp
+      end
+      Thread.main.raise local_exp
+      nil
+    end
+
+    # debug print
+    def stdout_write(str)
+      @stdout_mutex.synchronize do
+	$stdout.write(str)
+      end
+    end
+
+    # external module loading
+    def self.def_fairy_interface(mod)
+      include mod
+    end
+  end
+
+  class FilterChain
+    def initialize(input)
+      @filters = [input]
+    end
+
+    def [](idx)
+      @filters[idx]
+    end
+
+    def show_chain
+      @filters.each_with_index{|f, idx|
+        puts "[#{idx}]\t#{f.class}" 
+      }
+    end
+
+    def method_missing(msg, *args, &block)
+      #pp msg
+      ret = @filters.last.__send__(msg, *args, &block)
+      if ret.kind_of?(Job)
+        @filters << ret
+        self
+      else
+        ret
+      end
+    end
+  end
+
+  def def_fairy_interface(mod)
+    ::Fairy::Fairy.instance_eval{include mod}
+  end
+  module_function :def_fairy_interface
+end
+
+require "fairy/client/filter"
+require "fairy/client/input"
+
+require "fairy/client/addins"
+

data/sample/grep.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require 'rubygems'
+require 'fairy'
+
+
+unless ARGV.size == 3
+  $stderr.puts "Usage: #{File.basename($0)} PATTERN INPUT OUTPUT"
+  exit(1)
+end
+
+pattern = Regexp.new(eval %{ "#{ARGV[0]}" })
+
+input_path = ARGV[1];
+output_path = ARGV[2];
+
+unless FileTest.exist?(input_path)
+  raise "Input doesn't exist."
+end
+
+if FileTest.exist?(output_path) && output_path.match(/\.vf\z/)
+  raise "Cannot override an existing VFile. Please confirm."
+end
+
+
+t0 = Time.now
+
+puts "[#{$$}] START: #{t0}"
+puts "[#{$$}]   pattern: #{pattern}"
+puts "[#{$$}]   input:  #{input_path}"
+puts "[#{$$}]   output: #{output_path}"
+
+fairy = Fairy::Fairy.new
+
+input = fairy.input input_path
+greped = input.grep(pattern, :ignore_exception => true)
+greped.output output_path
+
+t1 = Time.now
+puts "[#{$$}] DONE: #{t1} (#{t1-t0} sec)"
+
+

data/sample/ping.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'fairy'
+require 'yaml'
+
+yml_path = File.expand_path(File.dirname(__FILE__) + "/../tools/cap_recipe/cluster.yml")
+cluster = YAML.load_file(yml_path)
+
+fairy = Fairy::Fairy.new
+input = fairy.exec(cluster['nodes'].map{|n| "file://#{n}/"})
+map = input.map(%q{|uri| "#{`hostname -f`.chomp} (#{`hostname -i`.chomp}) is alive."})
+map.here.each{|responce| puts responce}
+
+

data/sample/sort.rb

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'fairy'
+require 'optparse'
+
+
+opt = {:k => 0, :t => /\s+/}
+
+op = OptionParser.new
+op.on('-k', '--key=POS', Integer){|v| opt[:k] = v}
+op.on('-n', '--numeric-sort'){|v| opt[:n] = v}
+op.on('-r', '--reverse', nil, "This must be used with -n."){|v| opt[:r] = v}
+op.on('-t', '--separator=SEPARATOR'){|v| opt[:t] = v}
+op.parse!(ARGV)
+
+
+unless ARGV.size == 2
+  $stderr.puts op.to_s.sub(/ *\n/, "... INPUT OUTPUT\n")
+  exit(1)
+end
+
+input_path = ARGV[0];
+output_path = ARGV[1];
+
+unless FileTest.exist?(input_path)
+  raise "Input doesn't exist."
+end
+
+if FileTest.exist?(output_path) && output_path.match(/\.vf\z/)
+  raise "Cannot override an existing VFile. Please confirm."
+end
+
+if opt[:r] && !opt[:n]
+  raise "Cannot specify -r option without -n."
+end
+
+if opt[:t].is_a? Regexp
+  sep = opt[:t].to_s
+else 
+  str = eval %{ "#{opt[:t]}" }
+  if str.size == 1 && str != "\\"
+    sep = (/(?<!\\)#{str}/o).to_s
+  else
+    sep = str
+  end
+end
+
+t0 = Time.now
+
+puts "[#{$$}] START: #{t0}"
+puts "[#{$$}]   input:  #{input_path}"
+puts "[#{$$}]   output: #{output_path}"
+puts "[#{$$}]   key: #{opt[:k]}"
+puts "[#{$$}]   separator: #{sep}"
+puts "[#{$$}]   num-sort: ON" if opt[:n]
+puts "[#{$$}]   reverse: ON" if opt[:r]
+
+
+fairy = Fairy::Fairy.new
+
+fairy.def_pool_variable(:errors, :block => %{Array.new})
+
+input = fairy.input input_path
+maped = input.map(%{|ln| 
+  @sep_re ||= Regexp.new(#{sep.inspect})
+  begin
+    sort_key = ln.split(@sep_re)[#{opt[:k]}]
+    [sort_key, ln]
+  rescue => e
+    @Pool.errors.push([e.message, ln])
+    Import::TOKEN_NULLVALUE
+  end
+})
+
+if opt[:n] && opt[:r]
+  sorted = maped.sort_by(%{|ary| -ary[0].to_i})
+elsif opt[:n]
+  sorted = maped.sort_by(%{|ary| ary[0].to_i})
+else
+  sorted = maped.sort_by(%{|ary| ary[0]})
+end
+
+formatted = sorted.map(%{|ary| ary[1]})
+formatted.output output_path
+
+unless fairy.pool_variable(:errors).size.zero?
+  err = fairy.pool_variable(:errors)
+  puts "[#{$$}] WARN: #{err.size} error(s) occurred."
+  err.each_with_index{|e,i|
+    p [i+1] + e
+  }
+end
+
+t1 = Time.now
+puts "[#{$$}] DONE: #{t1} (#{t1-t0} sec)"
+
+

data/sample/wordcount.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'fairy'
+
+
+unless ARGV.size == 2
+  $stderr.puts "Usage: #{File.basename($0)} INPUT OUTPUT"
+end
+
+input_path = ARGV[0];
+output_path = ARGV[1];
+
+unless FileTest.exist?(input_path)
+  raise "Input doesn't exist."
+  exit(1)
+end
+
+if FileTest.exist?(output_path) && output_path.match(/\.vf\z/)
+  raise "Cannot override an existing VFile. Please confirm."
+end
+
+t0 = Time.now
+
+puts "[#{$$}] START: #{t0}"
+puts "[#{$$}]   input:  #{input_path}"
+puts "[#{$$}]   output: #{output_path}"
+
+fairy = Fairy::Fairy.new
+
+fairy.def_pool_variable(:errors, :block => %{Array.new})
+
+input = fairy.input input_path
+maped = input.mapf(%{|ln| 
+  begin
+    ln.split
+  rescue => e
+    @Pool.errors.push([e.message, ln])
+    Import::TOKEN_NULLVALUE
+  end
+})
+grouped = maped.group_by(%{|w| w})
+counted = grouped.map(%q{|bag| "#{bag.key}\t#{bag.size}"})
+counted.output output_path
+
+unless fairy.pool_variable(:errors).size.zero?
+  err = fairy.pool_variable(:errors)
+  puts "[#{$$}] WARN: #{err.size} error(s) occurred."
+  err.each_with_index{|e,i|
+    p [i+1] + e
+  }
+end
+
+t1 = Time.now
+puts "[#{$$}] DONE: #{t1} (#{t1-t0} sec)"
+
+