fairy 0.6.0

data/lib/fairy/client/exec.rb

@@ -0,0 +1,54 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/vfile"
+
+module Fairy
+  class Exec < Filter
+    module Interface
+
+      # Usage:
+      # fairy.exec(vnode-spec)....
+      #
+      def exec(vnode_spec, opts={})
+	Exec.exec(self, opts, vnode_spec)
+      end
+    end
+    Fairy::def_fairy_interface Interface
+
+    def Exec.exec(fairy, opts, vnode_spec)
+      exec = new(fairy, opts)
+      exec.start(vnode_spec)
+      exec
+    end
+
+
+    def backend_class_name
+      "CExec"
+    end
+
+    def start(vnode_spec)
+      @vnode_spec = vnode_spec
+      case @vnode_spec
+      when Array
+	vf = VFile.real_files(@vnode_spec)
+      when VFile
+	vf = @vnode_spec
+      when String
+	if VFile.vfile?(@vnode_spec)
+	  vf = VFile.vfile(@vnode_spec)
+	else
+	  vf = VFile.real_files([@vnode_spec])
+	end
+      else
+	ERR::Raise ERR::IllegalVFile
+      end
+
+      backend.start(vf)
+    end
+
+  end
+end

data/lib/fairy/client/filter.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+
+  FilterInterfaces = []
+
+  def self.def_filter_interface(mod)
+    FilterInterfaces.push mod
+    Filter.instance_eval{include mod}
+  end
+
+  def Fairy.def_filter_interface(mod)
+    ::Fairy.def_filter_interface(mod)
+  end
+
+  @PostInitializers = []
+  def self.def_post_initialize(&block)
+    @PostInitializers.push block
+  end
+
+  def self.post_initialize
+    @PostInitializers.each{|proc| proc.call}
+  end
+
+  class Filter
+    def initialize(fairy, opts, *rests)
+      @fairy = fairy
+      @opts = opts
+      @opts = {} unless @opts
+      if @opts[:BEGIN]
+	@opts[:BEGIN] = BlockSource.new(@opts[:BEGIN])
+      end
+      if @opts[:END]
+	@opts[:END] = BlockSource.new(@opts[:END])
+      end
+      @ref = backend_class.new(fairy.controller, opts, *rests)
+    end
+
+    def backend_class
+      unless klass = @fairy.name2backend_class(backend_class_name)
+	ERR::Raise ERR::INTERNAL::NoRegisterService, backend_class_name
+      end
+      klass
+    end
+
+    def backend
+      @ref
+    end
+
+    def backend=(v)
+      @ref=v
+    end
+
+    def def_pool_variable(vname, value = nil)
+      backend.def_pool_variable(vname, value)
+    end
+
+  end
+end

data/lib/fairy/client/find.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class Find<IOFilter
+
+    module Interface
+      # filter.find(%{...})
+      def find(block_source, opts = nil)
+	block_source = BlockSource.new(block_source) 
+	find = Find.new(@fairy, opts, block_source)
+	find.input = self
+	find
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CFind"
+    end
+
+    def value
+      backend.value
+    end
+  end
+end

data/lib/fairy/client/group-by.rb

@@ -0,0 +1,194 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/basic-group-by"
+require "fairy/client/merge-group-by"
+
+module Fairy
+  class GroupBy<IOFilter
+
+    module Interface
+      def group_by(hash_block, opts = nil)
+	hash_block = BlockSource.new(hash_block) 
+	mod_group_by = GroupBy.new(@fairy, opts, hash_block)
+	mod_group_by.input = self
+	mod_group_by
+      end
+    end
+    Fairy::def_filter_interface Interface
+    ::Fairy::def_post_initialize{post_initialize}
+
+    UnhandleMethods = [
+      :post_mod_group_by_filter,
+      :post_merge_group_by_filter
+    ]
+    def self.post_initialize
+      for interface in ::Fairy::FilterInterfaces
+	for m in interface.instance_methods
+	  m = m.intern if m.kind_of?(String)
+	  next if UnhandleMethods.include?(m)
+	
+	  m = m.id2name
+	  GroupBy::module_eval %{
+            def #{m}(*argv, &block)
+	      post_mod_group_by_filter(@block_source, @opts).#{m}(*argv, &block)
+	    end
+          }
+	end
+      end
+    end
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CGroupBy"
+    end
+
+    class PostFilter<IOFilter
+      module Interface
+	def post_mod_group_by_filter(hash_block, opts = nil)
+	  post_mod_group_by_filter = PostFilter.new(@fairy, opts, hash_block)
+	  post_mod_group_by_filter.input = self
+	  post_mod_group_by_filter
+	end
+	Fairy::def_filter_interface Interface
+      end
+
+      def initialize(fairy, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def backend_class_name
+	"CGroupBy::CPostFilter"
+      end
+    end
+
+#   class PostAfterModFilter<Filter
+#     module Interface
+#       def post_after_mod_filter(hash_block, opts = nil)
+# 	post_after_mod_filter = PostAfterModFilter.new(@fairy, opts, hash_block)
+# 	post_after_mod_filter.input = self
+# 	post_after_mod_filter
+#       end
+#     end
+#     Fairy::def_filter_interface Interface
+
+#     def initialize(fairy, opts, block_source)
+#       super
+#       @block_source = block_source
+#     end
+
+#     def backend_class_name
+#       "BPostAfterModFilter"
+#     end
+#   end
+  
+  end
+end
+
+
+Fairy.def_filter(:mod_group_by2) do |fairy, input, block_source, opts = {}|
+  my_begin = %{
+    require CONF.GROUP_BY_HASH_MODULE
+    hash = Fairy::HValueGenerator.new(@Pool.HASH_SEED)
+    mod = CONF.GROUP_BY_NO_SEGMENT
+  }
+  if opts[:BEGIN]
+    opts[:BEGIN].cat my_begin
+  else
+    opts[:BEGIN] = my_begin
+  end
+  if opts[:postqueuing_policy]
+    if !opts[:postqueuing_policy][:sort_by]
+      opts[:postqueuing_policy][:sort_by] = block_source
+    end
+  else
+    opts[:postqueuing_policy] = {
+      :queuing_class => :SortedQueue, 
+      :sort_by => block_source
+    }
+  end
+
+  pre = input.merge_group_by(%{|e| hash.value(proc{#{block_source}}.call(e)) % mod}, 
+			     opts)
+  post = pre.seg_map(%{|i, block|
+    sort_proc = proc{#{block_source}}
+
+    key = nil
+    ary = []
+    buf = i.map{|st| [st, st.pop.dc_deep_copy]}.select{|st, v|!v.nil?}.sort_by{|st, v| sort_proc.call(v)}
+    while st_min = buf.shift
+      st, min = st_min
+      if key == sort_proc.call(min)
+         ary.push min
+      else
+         block.call ary unless ary.empty?
+         key = sort_proc.call(min)
+         ary = [min]
+      end
+      next unless v = st.pop.dc_deep_copy # 取りあえずの対応
+      buf.push [st, v]
+      buf = buf.sort_by{|st0, v0| sort_proc.call(v0)}
+    end
+    if !ary.empty?
+      block.call [key, ary]
+    end
+    })
+end
+
+Fairy.def_filter(:mod_group_by3) do |fairy, input, block_source, opts = {}|
+
+  my_begin = %{
+    require CONF.GROUP_BY_HASH_MODULE
+    hash = Fairy::HValueGenerator.new(@Pool.HASH_SEED)
+    mod = CONF.GROUP_BY_NO_SEGMENT
+  }
+  if opts[:BEGIN]
+    opts[:BEGIN].cat my_begin
+  else
+    opts[:BEGIN] = my_begin
+  end
+  if opts[:postqueuing_policy]
+    if !opts[:postqueuing_policy][:sort_by]
+      opts[:postqueuing_policy][:sort_by] = %{|k, v| k}
+    end
+  else
+    opts[:postqueuing_policy] = {
+      :queuing_class => :SortedQueue, 
+      :sort_by => %{|k, v| k}
+    }
+  end
+
+  key_pair = input.map(%{|v| [proc{#{block_source}}.call(v), v]})
+  pre = key_pair.merge_group_by(%{|k, v| hash.value(k) % mod}, opts)
+  post = pre.smap2(%{|i, block|
+    key = nil
+    ary = []
+    buf = i.map{|st| [st, st.pop.dc_deep_copy]}.select{|st, pair|!pair.nil?}.sort_by{|st, pair| pair.first}
+    while st_min = buf.shift
+      st, min_pair = st_min
+      if key == min_pair.first
+         ary.push min_pair.last
+      else
+         block.call [key, ary] unless ary.empty?
+         key = min_pair.first
+         ary = [min_pair.last]
+      end
+      next unless v = st.pop.dc_deep_copy # 取りあえずの対応
+      buf.push [st, v]
+      buf = buf.sort_by{|st0, v0| v0.first}
+    end
+    if !ary.empty?
+      block.call ary
+    end
+    },
+		   :postqueuing_policy => {:queuing_class => :OnMemoryQueue}
+)
+  
+end

data/lib/fairy/client/here.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+require "fairy/share/port"
+
+module Fairy
+  class Here<IOFilter
+    include Enumerable
+
+    module Interface
+      def here(opts = nil)
+	here = Here.new(@fairy, opts)
+	here.input= self
+	here
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts = nil)
+      super
+    end
+
+    def backend_class_name
+      "CHere"
+    end
+
+#     def each(&block)
+#       backend.each{|e| block.call e}
+#     end
+
+    def each(&block)
+      policy = @opts[:prequeuing_policy]
+      
+      imports = Queue.new
+
+      parent_thread = Thread.current
+      
+      Thread.start do
+	begin
+	  backend.each_node do |node|
+	    node.start_export
+	    import = Import.new(policy)
+	    import.set_log_callback do |n, key| 
+	      Log::verbose(self, "IMPORT POP key=#{key}: #{n}")
+	    end
+	    import.no_import = 1
+	    node.export.output = import
+	    imports.push import
+	    nil # 消すな!!(BUG#250対応)
+	  end
+	  imports.push nil
+	rescue Exception
+	  parent_thread.raise $!
+	end
+      end
+
+      while imp = imports.pop
+	imp.each do |e|
+	  block.call e
+	end
+      end
+    end
+
+    def each_with_bjobeach(&block)
+      backend.each_buf do |buf|
+	buf.each &block
+	# GCの問題
+	buf = nil
+      end
+    end
+
+    def to_a
+      ary = []
+      backend.each{|e| ary.push e}
+      ary
+    end
+
+  end
+
+#  class BHere;end
+end

data/lib/fairy/client/inject.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class Inject<IOFilter
+
+    module Interface
+      # filter.inject(%{...}, :init_value = val)
+      def inject(block_source, opts = nil)
+	block_source = BlockSource.new(block_source) 
+	inject = Inject.new(@fairy, opts, block_source)
+	inject.input = self
+	#DeepConnect::future{inject.value}
+	inject
+      end
+
+      def min(block_source=nil, opts = nil)
+	unless block_source
+	  block_source = %{|e1, e2| e1 <=> e2}
+	end
+	
+	inject(%{|r, v| (proc{#{block_source}}.call(r, v) < 0) ? r : v})
+      end
+
+      def min_by(block_source, opts = nil)
+	pair = map(%{|v| [proc{#{block_source}}.call(v), v]})
+	min_by = pair.inject(%{|r, v| ((r[0] <=> v[0]) < 0) ? r : v})
+	def min_by.value
+	  super[1]
+	end
+	min_by
+      end
+
+      def max(block_source=nil, opts = nil)
+	unless block_source
+	  block_source = %{|e1, e2| e1 <=> e2}
+	end
+	
+	inject(%{|r, v| (proc{#{block_source}}.call(r, v) < 0) ? v : r})
+      end
+
+      def max_by(block_source, opts = nil)
+	pair = map(%{|v| [proc{#{block_source}}.call(v), v]})
+	max_by = pair.inject(%{|r, v| ((r[0] <=> v[0]) < 0) ? v : r})
+	def max_by.value
+	  super[1]
+	end
+	max_by
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CInject"
+    end
+
+    def value
+      backend.value
+    end
+  end
+end