fairy 0.6.0

data/lib/fairy/client/seg-join.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class SegJoin<IOFilter
+
+    module Interface
+      # jpb.seg_join(opts,...,filter,...,block_source, opts,...)
+      def seg_join(*others)
+	block_source = nil
+	if others.last.kind_of?(String)
+	  block_source = others.pop
+	elsif others.last.kind_of?(Hash) and others[-2].kind_of?(String)
+	  block_source = others.delete_at(-2)
+	end
+	others, opts = others.partition{|e| e.kind_of?(Filter)}
+	if opts.last.kind_of?(Hash)
+	  h = opts.pop
+	else
+	  h = {}
+	end
+	opts.each{|e| h[e] = true}
+
+	pres = others.collect{|o|
+	  p = PreJoinedFilter.new(@fairy, h)
+	  p.input = o
+	  p
+	}
+
+	block_source = BlockSource.new(block_source) 
+	join = SegJoin.new(@fairy, h, pres, block_source)
+	join.input = self
+	join
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, others, block_source)
+      super(fairy, opts, others.collect{|o| o.backend}, block_source)
+      @others = others
+      @block_source
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CSegJoin"
+    end
+
+    class PreJoinedFilter<IOFilter
+      def backend_class_name
+	"CSegJoin::CPreSegJoinFilter"
+      end
+    end
+
+  end
+end
+

data/lib/fairy/client/seg-map.rb

@@ -0,0 +1,78 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+
+  class SegMap<IOFilter
+    module Interface
+
+      def smap(block_source, opts = nil)
+	raise "No compatibility after fairy-0.5"
+      end
+
+      def smap2(block_source, opts = nil)
+	raise "No compatibility after fairy-0.6"
+      end
+
+      def seg_map(block_source, opts = nil)
+	ERR::Raise ERR::CantAcceptBlock if block_given?
+	block_source = BlockSource.new(block_source) 
+	mapper = SegMap.new(@fairy, opts, block_source)
+	mapper.input=self
+	mapper
+      end
+
+      # emap(%{|input| input.collect{..})
+      def emap(block_source, opts = nil)
+	ERR::Raise ERR::CantAcceptBlock if block_given?
+	map_source = %{|i, block| proc{#{block_source}}.call(i).each{|e| block.call e}}
+	seg_map(map_source, opts)
+      end
+
+      def map_flatten(block_source, opts = nil)
+	ERR::Raise ERR::CantAcceptBlock if block_given?
+	map_source = %{|i, block|
+          i.each do |e|
+            enum = proc{#{block_source}}.call(e)
+            enum.each do |f|
+              #{n = opts && opts[:N]; n ||= 1
+                case n
+                when 1
+                  "block.call f"
+                when 2
+                  "if f.respond_to?(:each)
+                     f.each{|g| block.call(g)}
+                   else
+                     block.call f
+                   end"
+                else
+                 "if f.respond_to?(:flatten)
+                    f.flatten(#{opts[:N]} - 2).each{|g| block.call(g)}
+                  else
+                    block.call f
+                  end"
+                end}
+            end
+          end
+        }
+	seg_map(map_source, opts)
+      end
+      alias mapf map_flatten
+
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CSegMap"
+    end
+  end
+end

data/lib/fairy/client/seg-shuffle.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class SegShuffle<IOFilter
+    module Interface
+      def seg_shuffle(block_source, opts = nil)
+	block_source = BlockSource.new(block_source) 
+	shuffle = SegShuffle.new(@fairy, opts, block_source)
+	shuffle.input = self
+	shuffle
+      end
+
+      def seg_eshuffle(block_source, opts = nil)
+	map_source = %{|i, o| proc{#{block_source}}.call(i).each{|e| o.push e}}
+	seg_shuffle(map_source, opts)
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CSegShuffle"
+    end
+  end
+end

data/lib/fairy/client/seg-split.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class SegSplit<IOFilter
+    module Interface
+      def seg_split(n, opts=nil)
+	splitter = SegSplit.new(@fairy, opts, n)
+	splitter.input = self
+	splitter
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+
+    def initialize(fairy, opts, n)
+      super
+      @no_split = n
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CSegSplit"
+    end
+  end
+end

data/lib/fairy/client/seg-zip.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class SegZip<IOFilter
+
+    module Interface
+      # jpb.seg_zip(opts,...,filter,...,block_source, opts,...)
+      def seg_zip(*others)
+	block_source = nil
+	if others.last.kind_of?(String)
+	  block_source = others.pop
+	end
+	others, opts = others.partition{|e| e.kind_of?(Filter)}
+	if opts.last.kind_of?(Hash)
+	  h = opts.pop
+	else
+	  h = {}
+	end
+	opts.each{|e| h[e] = true}
+
+	pres = others.collect{|o|
+	  p = PreSegZipFilter.new(@fairy, h)
+	  p.input = o
+	  p
+	}
+
+	block_source = BlockSource.new(block_source) 
+	zip = SegZip.new(@fairy, h, pres, block_source)
+	zip.input = self
+	zip
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+#    ZIP_BY_SEGMENT = :ZIP_BY_SEGMENT
+
+    def initialize(fairy, opts, others, block_source)
+      super(fairy, opts, others.collect{|o| o.backend}, block_source)
+      @others = others
+      @block_source
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CSegZip"
+    end
+
+    class PreSegZipFilter<IOFilter
+      def backend_class_name
+	"CSegZip::CPreSegZipFilter"
+      end
+    end
+  end
+end
+

data/lib/fairy/client/select.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+
+  class Select<IOFilter
+    module Interface
+      def select(block_source, opts = nil)
+	ERR::Raise ERR::CantAcceptBlock if block_given?
+	block_source = BlockSource.new(block_source) 
+	mapper = Select.new(@fairy, opts, block_source)
+	mapper.input=self
+	mapper
+      end
+
+      alias find_all select
+
+      def grep(regexp, opts = nil)
+	select(%{|e| /#{regexp.source}/ === e}, opts)
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CSelect"
+    end
+  end
+
+end

data/lib/fairy/client/sort.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/basic-group-by"
+
+module Fairy
+
+  module Sort
+
+    module Interface
+      def sort_by(cmp_proc, opts=nil)
+	cmp_proc = BlockSource.new(cmp_proc)
+	pre_sort = Sort::PreSort.new(@fairy, opts, cmp_proc)
+	pre_sort.input = self
+	post_sort = Sort::PostSort.new(@fairy, opts, cmp_proc)
+	post_sort.input = pre_sort
+	post_sort
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    class PreSort<IOFilter
+      def initialize(fairy, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def backend_class_name
+	"CSort::CPreSort"
+      end
+    end
+
+    class PostSort<IOFilter
+      def initialize(fairy, opts, block_source)
+	super
+	@block_source = block_source
+      end
+
+      def backend_class_name
+	"CSort::CPostSort"
+      end
+    end
+
+  end
+end
+

data/lib/fairy/client/sort18.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+
+require "fairy/client/merge-group-by"
+
+Fairy.def_filter(:sort_by_with_va) do |fairy, input, block_source, *rests|
+
+  opts = {}
+  if !rests.empty?
+    opts = rests.last
+  end
+  
+  sampling_ratio_1_to = opts[:sampling_ratio]
+  sampling_ratio_1_to ||= Fairy::CONF.SORT_SAMPLING_RATIO_1_TO
+  pvn = opts[:pvn]
+  pvn ||= Fairy::CONF.SORT_NO_SEGMENT
+  
+  va = input.emap(%{|i| 
+    sort_proc = proc{#{block_source}}
+    i.to_a.collect{|e| [sort_proc.call(e), e]}.sort_by{|e| e.first}}).to_va
+
+  if va.size/sampling_ratio_1_to < Fairy::CONF.SORT_SAMPLING_MIN
+    sampling_ratio_1_to = Fairy::CONF.SORT_SAMPLING_MIN.div(va.size)
+  end
+  if va.size/sampling_ratio_1_to > Fairy::CONF.SORT_SAMPLING_MAX
+    sampling_ratio_1_to = Fairy::CONF.SORT_SAMPLING_MAX.div(va.size)
+  end
+
+  Fairy::Log::debug(self, "SAMPLING: RATIO: 1/#{sampling_ratio_1_to}")
+  sample = fairy.input(va).select(%{|e| (i += 1) % #{sampling_ratio_1_to} == 0},
+				  :BEGIN=>%{i = 0}).here.sort_by{|e| e.first}.map{|e| e.first}
+
+  idxes = (1...pvn).collect{|i| (sample.size*i).div(pvn)}
+  idxes.push -1
+  pvs = sample.values_at(*idxes)
+  Fairy::Log::debug(self, "PVS: #{pvs.inspect}")
+  fairy.def_pool_variable(:pvs, pvs)
+
+  div = fairy.input(va).merge_group_by(%{|e| 
+    key = @Pool.pvs.find{|pv| e.first <= pv}
+    key ? key : @Pool.pvs.last},
+				       :postqueuing_policy => {:queuing_class => :OnMemoryQueue})
+
+  msort = div.seg_map(%{|i, o|
+    buf = i.map{|st| [st, st.pop.dc_deep_copy]}.select{|st, v|!v.nil?}.sort_by{|st, v| v.first}
+    while st_min = buf.shift
+      st, min = st_min
+      o.push min.last
+      next unless v = st.pop.dc_deep_copy # 取りあえずの対応
+      idx = buf.rindex{|st0, v0| v0.first <= v.first}
+      idx ? buf.insert(idx+1, [st, v]) : buf.unshift([st, v])
+    end})
+  
+  shuffle = msort.seg_eshuffle(%{|i| i.sort{|s1, s2| s1.key <=> s2.key}})
+#  shuffle = msort.eshuffle(%{|i| i.sort_by{|s1| Log::debug(self, s1.key.inspect); s1.key}})
+end
+

data/lib/fairy/client/sort19.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/merge-group-by"
+
+Fairy.def_filter(:sort_by_with_va, :sub => true) do |fairy, input, block_source, opts = {}|
+  
+  sampling_ratio_1_to = opts[:sampling_ratio]
+  sampling_ratio_1_to ||= Fairy::CONF.SORT_SAMPLING_RATIO_1_TO
+  pvn = opts[:pvn]
+  pvn ||= Fairy::CONF.SORT_NO_SEGMENT
+  
+  va = input.emap(%{|i| 
+    sort_proc = proc{#{block_source}}
+    i.to_a.collect{|e| [sort_proc.call(e), e]}.sort_by{|e| e.first}}).to_va
+
+  if va.size/sampling_ratio_1_to < Fairy::CONF.SORT_SAMPLING_MIN
+    #sampling_ratio_1_to = Fairy::CONF.SORT_SAMPLING_MIN.div(va.size)
+    sampling_ratio_1_to = va.size.div(Fairy::CONF.SORT_SAMPLING_MIN)
+    sampling_ratio_1_to = 1 if sampling_ratio_1_to.zero?
+  end
+  if va.size/sampling_ratio_1_to > Fairy::CONF.SORT_SAMPLING_MAX
+    sampling_ratio_1_to = va.size.div(Fairy::CONF.SORT_SAMPLING_MAX)
+  end
+
+  Fairy::Log::debug(self, "SAMPLING: RATIO: 1/#{sampling_ratio_1_to}")
+  Fairy::Log::debug(self, "SAMPLING: VA SIZE: #{va.size}")
+  sample = fairy.input(va).select(%{|e| (i += 1) % #{sampling_ratio_1_to} == 0},
+				  :BEGIN=>%{i = 0}).here.sort_by{|e| e.first}.map{|e| e.first}
+
+  Fairy::Log::debugf(self, "SAMPLING: SAMPLE: %s", sample.inspect)
+
+  idxes = (1...pvn).collect{|i| (sample.size*i).div(pvn)}
+  idxes.push -1
+  pvs = sample.values_at(*idxes)
+  Fairy::Log::debug(self, "PVS: #{pvs.inspect}")
+  fairy.def_pool_variable(:pvs, pvs)
+
+  div = fairy.input(va).merge_group_by(%{|e| 
+    key = @Pool.pvs.find{|pv| e.first <= pv}
+    key ? key : @Pool.pvs.last},
+				       :postqueuing_policy => {:queuing_class => :PoolQueue}
+
+)
+
+  msort = div.seg_map(%{|i, block|
+    buf = i.map{|st| [st, st.pop.dc_deep_copy]}.select{|st, v|!v.nil?}.sort_by{|st, v| v.first}
+    while st_min = buf.shift
+      st, min = st_min
+      block.call min.last
+      next unless v = st.pop.dc_deep_copy # 取りあえずの対応
+      idx = buf.rindex{|st0, v0| v0.first <= v.first}
+      idx ? buf.insert(idx+1, [st, v]) : buf.unshift([st, v])
+    end})
+  
+  shuffle = msort.seg_eshuffle(%{|i| i.sort{|s1, s2| s1.key <=> s2.key}})
+#  shuffle = msort.eshuffle(%{|i| i.sort_by{|s1| Log::debug(self, s1.key.inspect); s1.key}})
+end
+