fairy 0.6.0

data/lib/fairy/client/merge-group-by.rb

@@ -0,0 +1,71 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/basic-group-by"
+
+module Fairy
+  class MergeGroupBy<IOFilter
+
+    module Interface
+      def merge_group_by(hash_block, opts = nil)
+	hash_block = BlockSource.new(hash_block) 
+	merge_group_by = MergeGroupBy.new(@fairy, opts, hash_block)
+	merge_group_by.input = self
+	merge_group_by
+      end
+    end
+    Fairy::def_filter_interface Interface
+#    ::Fairy::Def_Post_Initialize{Post_Initialize}
+
+#     Unhandlemethods = [
+#       :Post_Mod_Group_By_Filter,
+#       :Post_Merge_Group_By_Filter
+#     ]
+#     Def Self.Post_Initialize
+#       For Interface In ::Fairy::Jobinterfaces
+# 	For M In Interface.Instance_Methods
+# 	  M = M.Intern If M.Kind_Of?(String)
+# 	  Next If Unhandlemethods.Include?(M)
+	
+# 	  M = M.Id2name
+# 	  Mergegroupby::Module_Eval %{
+#             Def #{M}(*Argv, &Block)
+# 	      Post_Merge_Group_By_Filter(@Block_Source, @Opts).#{M}(*Argv, &Block)
+# 	    End
+#           }
+# 	End
+#       End
+#     End
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CMergeGroupBy"
+    end
+
+#     class PostFilter<Filter
+#       module Interface
+# 	def post_merge_group_by_filterr(hash_block, opts = nil)
+# 	  post_merge_group_by_filter = PostFilter.new(@fairy, opts, hash_block)
+# 	  post_merge_group_by_filter.input = self
+# 	  post_merge_group_by_filter
+# 	end
+# 	Fairy::def_filter_interface Interface
+#       end
+
+#       def initialize(fairy, opts, block_source)
+# 	super
+# 	@block_source = block_source
+#       end
+
+#       def backend_class_name
+# 	"BMergeGroupBy::BPostFilter"
+#       end
+#     end
+  end
+end

data/lib/fairy/client/output-file.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/vfile"
+
+module Fairy
+  class OutputFile < Filter
+
+    def OutputFile.output(fairy, opts, vfn)
+      ffile = new(fairy, opts)
+      ffile.output(vfn)
+      ffile
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+
+      @old_vfile = nil
+      @vfile = nil
+    end
+
+    def backend_class_name
+      "COutputFile"
+    end
+
+    def output(vfn)
+      if File.exists?(vfn)
+	@old_vfile = VFile.vfile(vfn)
+      end
+
+      @descripter = vfn
+      @vfile = VFile.new
+      @vfile.vfile_name = vfn
+      backend.output(@vfile)
+
+#      vf.create_vfile(vfn)
+    end
+
+    def input=(job)
+      @input = job
+      backend.input=job.backend
+      backend.wait_all_output_finished
+
+      if @old_vfile
+	rmfiles = @old_vfile.real_file_names.zip(@vfile.real_file_names).select{|n1, n2| n1 != n2}.map{|n1, n2| n1}
+	if !rmfiles.empty?
+	  rm = @fairy.exec(rmfiles).map(%{|uri|
+             path = URI(uri).path
+             begin
+               File.unlink(path)
+             rescue
+             end
+             }, :BEGIN => %{require "uri"})
+	  rm.done
+	end
+      end
+      @vfile.create_vfile
+    end
+  end
+
+end

data/lib/fairy/client/output-local-file.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/here"
+
+require "fairy/share/vfile"
+
+module Fairy
+  class OutputLocalFile<Here
+
+    def OutputLocalFile.output(fairy, opts, filename)
+      ffile = new(fairy, opts)
+      ffile.output(filename)
+      ffile
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+      @filename = nil      
+    end
+
+    def backend_class_name
+      "COutputLocalFile"
+    end
+
+    def output(filename)
+      @filename = filename
+#      backend.output(self)
+    end
+
+    def input=(job)
+      super
+      
+      File.open(@filename, "w") do |io|
+	each do |e|
+	  io.puts e
+	end
+      end
+    end
+
+#     def input=(job)
+#       @input = job
+#       backend.input=job.backend
+      
+#       File.open(@filename, "w") do |io|
+# 	backend.each_buf do |buf|
+# 	  buf.each do |l|
+# 	    io.puts l
+# #	    l = nil             # 効果無し
+# 	  end
+# 	  # GCの問題[BUG: #135]
+# 	  buf.clear             # 50/200
+# 	  buf = nil             # 59/200
+# 	end
+#       end
+#     end
+  end
+end

data/lib/fairy/client/output-null.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/varray"
+
+module Fairy
+  class OutputNull<Filter
+    module Interface
+      # Usage:
+      # ... .done
+      #
+      def done
+	output_null = OutputNull.output(@fairy, opts=nil)
+	output_null.input = self
+	output_null
+      end
+      alias do done
+    end
+    Fairy::def_filter_interface Interface
+    
+    def self.output(fairy, opts)
+      output = new(fairy, opts)
+      output
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+    end
+
+    def backend_class_name
+      "COutputNull"
+    end
+
+    def output
+      backend.output
+    end
+
+    def input=(job)
+      @input = job
+      backend.input=job.backend
+      backend.wait_all_output_finished
+    end
+  end
+end

data/lib/fairy/client/output-varray.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/varray"
+
+module Fairy
+  class OutputVArray<Filter
+    module Interface
+      # Usage:
+      # ... .to_va
+      #
+      def to_va
+	output_va = OutputVArray.output(@fairy, opts=nil)
+	output_va.input = self
+	output_va.varray
+      end
+    end
+    Fairy::def_filter_interface Interface
+    
+    def self.output(fairy, opts)
+      output = new(fairy, opts)
+      output
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+
+      @varray = backend.varray
+    end
+
+    attr_reader :varray
+
+    def backend_class_name
+      "COutputVArray"
+    end
+
+    def output
+      backend.output
+    end
+
+    def input=(job)
+      @input = job
+      backend.input=job.backend
+#      backend.wait_all_output_finished
+    end
+  end
+end

data/lib/fairy/client/output.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  
+  module OutputInterface
+    def output(vfn, opts = nil)
+      if vfn.kind_of?(Class)
+	outputter = vfn.output(@fairy, opts)
+      elsif !vfn.kind_of?(String) || VFile.vfile?(vfn)
+	outputter = OutputFile.output(@fairy, opts, vfn)
+      else
+	outputter = OutputLocalFile.output(@fairy, opts, vfn)
+      end
+      outputter.input = self
+      outputter
+
+    end
+  end
+  def_filter_interface OutputInterface
+
+end
+
+require "fairy/client/output-file"
+require "fairy/client/output-local-file"
+
+

data/lib/fairy/client/roma-put.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/client/seg-map"
+
+module Fairy
+  module RomaPutInterface
+
+    DEFAULT_SPLIT = 100
+
+    def roma_put(ap, base_key, split=DEFAULT_SPLIT, opts={})
+      ap = [ap] if ap.kind_of?(String)
+
+      @fairy.def_pool_variable(:__ROMA_PUT_ap, ap)
+      @fairy.def_pool_variable(:__ROMA_PUT_base_key, base_key)
+      @fairy.def_pool_variable(:__ROMA_PUT_split, split)
+
+      if opts[:nice]
+	@fairy.def_pool_variable(:__ROMA_PUT_nice, opts[:nice])
+      else
+	@fairy.def_pool_variable(:__ROMA_PUT_nice, 0)
+      end
+
+      seg_map(%{|i, block|
+          system('renice '+@Pool.__ROMA_PUT_nice.to_s+' '+$$.to_s) unless @Pool.__ROMA_PUT_nice.zero?
+
+          require 'roma/client/rclient'
+
+          @roma = Roma::Client::RomaClient.new(@Pool.__ROMA_PUT_ap.dc_deep_copy)
+          @base_key = @Pool.__ROMA_PUT_base_key + $$.to_s + '_' + __id__.to_s
+          @split = @Pool.__ROMA_PUT_split
+
+          buf = []
+          cnt = 0
+          put_cnt = 0
+          i.each{|e|
+            cnt += 1
+            buf << e
+            if (cnt % @split == 0)
+              key = @base_key + ('%03d' % put_cnt)
+              @roma[key] = buf.join(',')
+              block.call(key)
+              buf.clear
+              put_cnt += 1
+            end
+          }
+          if buf.size > 0
+            key = @base_key + ('%03d' % put_cnt)
+            @roma[key] = buf.join(',')
+            block.call(key)
+            buf.clear
+          end
+        })
+    end
+
+    Fairy::def_filter_interface RomaPutInterface
+
+  end
+end

data/lib/fairy/client/roma.rb

@@ -0,0 +1,156 @@
+# encoding: UTF-8
+
+
+module Fairy
+  module RomaInterface
+    DUMP_KEY_PREFIX = '__dump_by_fairy__'
+    DEFAULT_HASH_NAME = 'roma'
+    DEFAULT_FORMAT    = [5, 7]
+
+    attr_reader :dump_id, :dump_key
+
+    #
+    # input data from ROMA
+    #
+    # ex) 
+    #   require 'fairy'
+    #  
+    #   fairy = Fairy::Fairy.new
+    #   f0 = fairy.roma(
+    #     'fairy-xm01_11211',
+    #     :format=>[5,7,1],
+    #     :select=>%{|k,v,t| v.to_i >= 500 && v.to_i < 600},
+    #     :map=>%{|k,v,t| [k.sub(/^key-/, ''), v, v.to_i, Time.at(t).strftime('%Y-%m-%d %H:%M:%S')]},
+    #     :nice=>20
+    #   )
+    #   f1 = f0.map(%{|*ary| ary.join(",")})
+    #   f1.output('./roma_test.out')
+    #
+    # available options are:
+    #   :hash     name of destination hash. '*' means all hashes (defalut='roma')
+    #   :format   fields of CSV. specify by Array (default=[5, 7])
+    #             available fields:
+    #               0: virtual node ID
+    #               1: last modified time (UNIX timestamp)
+    #               2: logical clock
+    #               3: expire time
+    #               4: key length
+    #               5: key
+    #               6: value length
+    #               7: value
+    #   :select   block source to select particular records
+    #   :map      block source to apply to each records
+    #   :nice     priority of process (LOW 20 <--> -20 HIGH) 
+    #
+    def roma(addr_port, opts={})
+      addr_port = [addr_port] if addr_port.kind_of?(String)
+
+      require 'roma/client/export/rdump'
+
+      @@roma_dump_serial_no ||= 0
+      @@roma_dump_serial_no += 1
+      @dump_id = "#{`hostname`.chomp}_#{Time.now.strftime("%y%m%d%H%M%S")}_#{$$}_#{@@roma_dump_serial_no}"
+      @dump_key = DUMP_KEY_PREFIX+dump_id
+      dumpinfo = Roma::Client::Export::RomaDump.dump(dump_key, dump_id, addr_port)
+
+      ##############################################
+      # structure of dumpinfo
+      # {
+      #   "hostname1_port1" => "STARTED path"
+      #   "hostname1_port2" => "STARTED path"
+      #   "hostname2_port1" => "STARTED path"
+      #   ...
+      # }
+      ##############################################
+
+      uris = []
+      dumpinfo.each{|host_port, stat_path|
+        m = host_port.match(/\A(.*)_\d+\z/)
+        host = m[1]
+        m2 = stat_path.match(/\A\S+\s+(.*)\z/)
+        path = m2[1]
+        uris << 'file://'+host+path
+      }
+
+      f0 = exec(uris)
+
+      if opts[:hash]
+        self.def_pool_variable(:dest_hash, opts[:hash])
+      else
+        self.def_pool_variable(:dest_hash, DEFAULT_HASH_NAME)
+      end
+
+      if opts[:nice]
+        self.def_pool_variable(:nice, opts[:nice])
+      else
+        self.def_pool_variable(:nice, 0)
+      end
+
+      f1 = f0.mapf(%q{|uri|
+        dumpfiles = Dir.glob(URI(uri).path+"/#{@dest_hash}/*.dump")
+        dumpfiles
+      }, :BEGIN => %q{
+        require 'uri'
+        @dest_hash = @Pool.dest_hash
+        system("renice #{@Pool.nice} #{$$}") unless @Pool.nice.zero?
+      })
+
+      if opts[:format]
+        self.def_pool_variable(:format, opts[:format])
+      else
+        self.def_pool_variable(:format, DEFAULT_FORMAT)
+      end
+
+      if opts[:select]
+        src = add_proc_src(opts[:select])
+        eval(src)  # syntax check
+        self.def_pool_variable(:select, src)
+      else
+        self.def_pool_variable(:select, '')
+      end
+
+      if opts[:map]
+        src = add_proc_src(opts[:map])
+        eval(src)  # syntax check
+        self.def_pool_variable(:map, src)
+      else
+        self.def_pool_variable(:map, '')
+      end
+
+      f2 = f1.mapf(%q{|dumpf|
+        records = []
+        File.open(dumpf, 'rb'){|io|
+          Roma::Client::Export::RomaDump.load(io) {|*rec|
+            rec_formatted = @format.inject([]){|res, idx| res << rec[idx]}
+            next if @select && !@select.call(*rec_formatted)
+            if @map
+              records << @map.call(rec_formatted)
+            else
+              records << rec_formatted
+            end
+          }
+        }
+        records
+      }, :N => 1, :BEGIN => %q{
+        require 'roma/client/export/rdump'
+        @format = @Pool.format.dc_deep_copy
+        @select = eval(@Pool.select) unless @Pool.select.empty?
+        @map = eval(@Pool.map) unless @Pool.map.empty?
+      })
+
+      f2  
+    end
+
+    private
+    def add_proc_src(src)
+      if src =~ /\A{.*}\z/
+        'Proc.new'+src
+      else
+        'Proc.new{'+src+'}'
+      end
+    end
+  end
+  def_fairy_interface RomaInterface
+end
+
+