fairy 0.6.0 → 0.6.5

data/lib/fairy/version.rb

@@ -3,6 +3,6 @@
 # This file is auto generation. 
 #
 module Fairy
-    Version = "0.6.0-001"
+    Version = "0.6.5-001"
 end
 

data/sample/sort.rb

@@ -7,6 +7,46 @@
 require 'rubygems'
 require 'fairy'
 require 'optparse'
+require 'json'
+require 'pp'
+
+
+def json2hash(str_or_hash)
+  if str_or_hash.is_a?(String)
+    h = JSON.parse(str_or_hash)
+  else
+    h = str_or_hash
+  end
+
+  ret = {}
+
+  h.each{|key,val|
+    key = key.to_sym
+    if val.is_a?(String) && val.start_with?(":")
+      v = val.to_sym
+    elsif val.is_a?(Hash)
+      v = json2hash(val)
+    else
+      v = val
+    end
+    ret[key.to_sym] = v
+  }
+
+  ret
+end
+
+def recursive_merge!(hash_a, hash_b)
+  hash_a.merge!(hash_b){|key,val_a,val_b|
+    if val_a.is_a?(Hash) && val_b.is_a?(Hash)
+      recursive_merge!(val_a, val_b)
+      val_a
+    elsif val_a.class == val_b.class
+      val_b
+    else
+      raise "Incompatible types: left=#{val_a.class}, right=#{val_b.class}"
+    end
+  }
+end
 
 
 opt = {:k => 0, :t => /\s+/}
@@ -16,6 +56,7 @@ op.on('-k', '--key=POS', Integer){|v| opt[:k] = v}
 op.on('-n', '--numeric-sort'){|v| opt[:n] = v}
 op.on('-r', '--reverse', nil, "This must be used with -n."){|v| opt[:r] = v}
 op.on('-t', '--separator=SEPARATOR'){|v| opt[:t] = v}
+op.on('-c', '--config=JSON'){|v| opt[:c] = json2hash(v)}
 op.parse!(ARGV)
 
 
@@ -59,6 +100,7 @@ puts "[#{$$}]   key: #{opt[:k]}"
 puts "[#{$$}]   separator: #{sep}"
 puts "[#{$$}]   num-sort: ON" if opt[:n]
 puts "[#{$$}]   reverse: ON" if opt[:r]
+puts "[#{$$}]   sort config: #{opt[:c].inspect}" if opt[:c]
 
 
 fairy = Fairy::Fairy.new
@@ -77,14 +119,38 @@ maped = input.map(%{|ln|
   end
 })
 
+sort_config = {
+  :postqueuing_policy => {
+    :queuing_class => :XMarshaledQueue,
+    :chunk_size => 10_000,
+    :buffers_cache_limit => 10
+  },
+  :postfilter_prequeuing_policy => {
+    :queuing_class => :XSizedQueue,
+    :queues_limit => 10
+  },
+  :buffering_policy => {
+    :buffering_class => "PGroupBy::XDirectMergeSortBuffer",
+    :threshold => 200_000,
+    :chunk_size => 1000
+  }
+}
+
+recursive_merge!(sort_config, opt[:c]) if opt[:c]
+
+#pp sort_config
+#exit
+
 if opt[:n] && opt[:r]
-  sorted = maped.sort_by(%{|ary| -ary[0].to_i})
+  sort_proc = %{|ary| -ary[0].to_i}
 elsif opt[:n]
-  sorted = maped.sort_by(%{|ary| ary[0].to_i})
+  sort_proc = %{|ary| ary[0].to_i}
 else
-  sorted = maped.sort_by(%{|ary| ary[0]})
+  sort_proc = %{|ary| ary[0]}
 end
 
+sorted = maped.sort_by(sort_proc, sort_config)
+
 formatted = sorted.map(%{|ary| ary[1]})
 formatted.output output_path
 

data/spec/fairy8_spec.rb

@@ -30,7 +30,7 @@ describe Fairy do
 
   # exec
   it 'should print all node-names' do
-    answer = @cluster["nodes"].sort
+    answer = @cluster["nodes"].map{|n| (n == "localhost") ? `hostname`.chomp : n}.sort
     
     result = []
     @fairy.exec(@cluster["nodes"].map{|n| "file://#{n}"}).map(%q{|uri|

data/test/testc.rb

@@ -2635,6 +2635,7 @@ when "55.1.4"
 #  		    "file://giant/home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_30M.txt"])
   f = f.mapf(%{|ln| begin
                       ln.chomp.split
+
 		    rescue
 		      []
 		    end
@@ -6328,8 +6329,8 @@ when "94.1"
 
 when "95"
 
-  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
-#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
 #  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
   f = f.mapf(%{|ln| begin
                       ln.chomp.split
@@ -7106,6 +7107,383 @@ when "113.0"
   f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*2)
   f.here.each{|e| puts e}
 
+when "114"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_120M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		 :no_segment => 1,
+		 :postqueuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000},
+		 :postfilter_prequeuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+#  f.here.each{|e| puts e}
+  f.output("test/test-114.vf")
+
+when "114.NS"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_120M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		 :no_segment => 1,
+		 :postqueuing_policy => {:queuing_class => :XMarshaledQueue},
+		 :postfilter_prequeuing_policy => {:queuing_class => :XMarshaledQueue},
+		 :use_string_buffer => false)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+#  f.here.each{|e| puts e}
+  f.output("test/test-114.vf")
+
+
+when "114.F"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_120M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		     :no_segment => 1,
+		     :postqueuing_policy => {:queuing_class => :FileMarshaledQueue},
+		     :postfilter_prequeuing_policy => {:queuing_class => :FileMarshaledQueue},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+#  f.here.each{|e| puts e}
+  f.output("test/test-114.vf")
+
+when "114.0"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  },		  
+	     :postmapping_policy => :MPNewProcessorN,
+	     :postqueuing_policy => {:queuing_class => :XMarshaledQueue},
+	     )
+#  f.here.each{|e| puts e}
+  f.output("test/test-114.vf",
+	   :prequeuing_policy => {:queuing_class => :XMarshaledQueue},
+)
+
+when "114.0.F"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"])
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  },		  
+	     :postmapping_policy => :MPNewProcessorN,
+	     :postqueuing_policy => {:queuing_class => :FileMarshaledQueue},
+	     )
+#  f.here.each{|e| puts e}
+  f.output("test/test-114.vf")
+
+when "114.0.H"
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"])
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  },		  
+	     :postmapping_policy => :MPNewProcessorN,
+	     :postqueuing_policy => {:queuing_class => :FileMarshaledQueue},
+	     )
+  f.here.each{|e| puts e}
+#  f.output("test/test-114.vf")
+
+when "115"
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		     :no_segment => 1,
+		 :postqueuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},
+		 :postfilter_prequeuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+when "116"
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.xgroup_by(%{|w| w},
+		     :no_segment => 1,
+		 :postqueuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},
+		 :postfilter_prequeuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+when "117"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		 :group_by => :XGroupBy,
+		 :no_segment => 1,
+		 :postqueuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},
+		 :buffering_policy => {
+		   :buffering_class => :DirectMergeSortBuffer,
+		   :threshold => 400_000},
+		 :postfilter_prequeuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :log_mstore => true,
+		   :buffers_cache_limit => 100},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+when "117.0"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w}, :no_segment => 1)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+when "118.0"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*2)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w}, :no_segment => 2)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+when "119"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f = f.group_by(%{|w| w},
+		 :group_by => :XGroupBy,
+		 :no_segment => 1,
+		 :postqueuing_policy => {
+		   :queuing_class => :XMarshaledQueue,
+		   :chunk_size => 10000,
+		   :buffers_cache_limit => 100},
+		 :buffering_policy => {
+		   :buffering_class => :XDirectMergeSortBuffer,
+		   :threshold => 400_000},
+		 :postfilter_prequeuing_policy => {
+		   :queuing_class => :XSizedQueue,
+		   :chunk_size => 10000,
+		   :cache_limit => 100},)
+  f = f.map(%{|values| [values.key, values.size].join(" ")})
+  f.output("test/test-pf.vf")
+  #  f.here.each{|e| puts e.join(" ")}
+
+
+when "200"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f.mapf(%{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums
+    }).group_by(%{|n| n.to_s}).map(%q{|bag|
+      "#{bag.key}\t#{bag.size}"
+    }).output("/tmp/fairy_spec_testdata_multi.txt")
+
+when "201"
+
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.map(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f.sort_by(%{|l| l[0] || ""}, 
+	    :no_segment=>12,
+	    :log_mstore=>true).map(%{|l| l.join("+")}).output("test/test.vf")
+
+when "201.1"
+
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.mapf(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f.sort_by(%{|l| l}, :no_segment=>24).output("test/test.vf")
+
+when "201.2"
+
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.map(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f.sort_by(%{|l| l[0] || ""}, 
+	    :no_segment => 12,
+	    :postqueuing_policy => {
+	      :queuing_class => :XMarshaledQueue,
+	      :chunk_size => 10_000,
+	      :buffers_cache_limit => 10,
+	    },
+	    :postfilter_prequeuing_policy => {
+	      :queuing_class => :XSizedQueue,
+	      :queues_limit => 10,
+	    },
+	    :buffering_policy => {
+	      :buffering_class => "PGroupBy::DirectMergeSortBuffer",
+	      :threshold => 400_000,
+	      :chunk_size => 1000,
+	    }).output("test/test-pf.vf")
+
+when "201.3"
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.map(%{|ln| begin
+                      ln.chomp.split
+		    rescue
+		      []
+		    end
+  })
+  f.sort_by(%{|l| l[0] || ""}, 
+	    :no_segment => 12,
+	    :postqueuing_policy => {
+	      :queuing_class => :XMarshaledQueue,
+	      :chunk_size => 10_000,
+	      :buffers_cache_limit => 10,
+	    },
+	    :postfilter_prequeuing_policy => {
+	      :queuing_class => :XSizedQueue,
+	      :queues_limit => 10,
+	    },
+	    :buffering_policy => {
+	      :buffering_class => "PGroupBy::XDirectMergeSortBuffer",
+	      :threshold => 200_000,
+	      :chunk_size => 1000,
+	    }).output("test/test-pf.vf")
+
+when "202", "BUG#316"
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.map(%{|ln| begin
+                      ln.chomp.split.size
+		    rescue
+		      0
+		    end
+  })
+  f.sort_by(%{|n| n.to_i}).map(%q{|n| "#{n}"}).output("/tmp/fairy-test202.txt")
+
+when "202.1", "BUG#316"
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_960M.txt"]*1)
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*1)
+#  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/fairy.cat"]*1)
+  f = f.map(%{|ln| begin
+                      ln.chomp.split.size
+		    rescue
+		      0
+		    end
+  })
+  f.sort_by(%{|n| -(n.to_i)}).map(%q{|n| "#{n}"}).output("/tmp/fairy-test202.txt")
+
+when "203"
+
+  f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_240M.txt"]*10)
+  #f = fairy.input(["file://emperor//home/keiju/public/a.research/fairy/git/fairy/sample/wc/data/sample_10M.txt"]*10)
+  i = 0
+  f.here(:prequeuing_policy => {
+	   :queuing_class => :XSizedQueue,
+	   :queues_limit => 1, }).each{i += 1}
+  p i
+
 end
 
 # test
+
+