fairy 0.6.0

data/spec/README

@@ -0,0 +1,12 @@
+Befor running tests here, 
+you need to edit $FARIY_HOME/tools/cap_recipe/cluster.yml
+along with your cluster.
+
+You can run all tests with following command:
+
+  ./run_all.sh
+
+
+Copyright (C) 2007-2010 Rakuten, Inc.
+
+

data/spec/fairy1_spec.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+require 'fairy/controller'
+
+describe Fairy do
+  before :all do
+    @fairy = Fairy::Fairy.new
+  end
+
+  # initialize
+  it 'should create new controller' do
+    @fairy.controller.kind_of?(Fairy::Controller).should be_true
+    @fairy.controller.deep_space.status.should == :SERVICING
+  end
+
+  # abort
+  it 'should destroy the controller' do
+    @fairy.abort
+    sleep(3)
+    @fairy.controller.deep_space.status.should == :SERVICE_STOP
+  end
+end
+
+

data/spec/fairy2_spec.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source_multi = File.readlines("testdata_multi.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # input/here
+  it 'should open a vfile & print the contents' do
+    contents = []
+    @fairy.input("testdata_multi.vf").here.each{|ln|
+      contents << ln
+    }
+    contents.should == @source_multi
+  end
+
+  # output (local)
+  it 'should output to a local file' do
+    @fairy.input("testdata_multi.vf").output("/tmp/fairy_spec_testdata_multi.txt")
+    contents = File.readlines("/tmp/fairy_spec_testdata_multi.txt")
+    contents.should == @source_multi
+  end
+
+  # output (vfile)
+  it 'should output to remote files (a vfile)' do
+    @fairy.input("testdata_multi.vf").output("/tmp/fairy_spec_testdata_multi.vf")
+    system %{ fairy cp /tmp/fairy_spec_testdata_multi.vf /tmp/fairy_spec_testdata_multi.txt }
+    contents = File.readlines("/tmp/fairy_spec_testdata_multi.txt")
+    contents.should == @source_multi
+  end
+end
+
+

data/spec/fairy3_spec.rb

@@ -0,0 +1,126 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source = File.readlines("testdata.txt")
+    @source_multi = File.readlines("testdata_multi.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # map (=collect)
+  it 'should strip LFs & convert values to integer & add one' do
+    answer = @source.map{|ln|
+      n = ln.chomp.to_i + 1
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata.vf").map(%q{|ln|
+      n = ln.chomp.to_i + 1
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+
+    @fairy.input("testdata.vf").collect(%q{|ln|
+      n = ln.chomp.to_i + 1
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+  end
+
+  # map + map
+  it 'should add one + double values' do
+    answer = @source.map{|ln|
+      n = ln.chomp.to_i + 1
+      n *= 2
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp.to_i + 1
+      n
+    }).map(%q{|n|
+      n *= 2
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+  end
+
+  # map + map + map
+  it 'should add one + double values + subtract three' do
+    answer = @source.map{|ln|
+      n = ln.chomp.to_i + 1
+      n *= 2
+      n -= 3
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp.to_i + 1
+      n
+    }).map(%{|n|
+      n *= 2
+      n
+    }).map(%q{|n|
+      n -= 3
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+  end
+
+  # map_flatten (=mapf)
+  it 'should split lines and flatten values' do
+    answer = @source_multi.map{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums
+    }.flatten.map{|n|
+      n + 1
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata_multi.vf").map_flatten(%{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums
+    }).map(%q{|n|
+      n + 1
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata_multi.txt")
+
+    result = File.readlines("/tmp/fairy_spec_testdata_multi.txt")
+
+    result.should == answer
+
+    @fairy.input("testdata_multi.vf").mapf(%{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums
+    }).map(%q{|n|
+      n + 1
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata_multi.txt")
+
+    result = File.readlines("/tmp/fairy_spec_testdata_multi.txt")
+
+    result.should == answer
+  end
+end
+
+

data/spec/fairy4_spec.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source = File.readlines("testdata.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # select
+  it 'should select multiples of 10' do
+    answer = @source.map{|ln|
+      n = ln.chomp.to_i
+      n
+    }.select{|n|
+      (n % 10).zero?
+    }.map{|n|
+      "#{n}\n"
+    }
+
+    result = []
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp.to_i
+      n
+    }).select(%{|n|
+      (n % 10).zero?
+    }).map(%q{|n|
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+  end
+
+  # grep
+  it 'should select 555' do
+    answer = @source.map{|ln|
+      n = ln.chomp
+      n
+    }.grep("555").map{|n|
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp
+      n
+    }).grep(/^555$/).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+  end
+end
+
+

data/spec/fairy5_spec.rb

@@ -0,0 +1,45 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source_multi = File.readlines("testdata_multi.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # mapf + group_by + map (word count)
+  it 'should count unique values' do
+    answer = {}
+    @source_multi.each{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums.each{|n|
+        answer[n] ||= 0
+        answer[n] += 1
+      }
+    }
+
+    @fairy.input("testdata_multi.vf").mapf(%{|ln|
+      nums = ln.split.map{|s| s.to_i}
+      nums
+    }).group_by(%{|n| n.to_s}).map(%q{|bag|
+      "#{bag.key}\t#{bag.size}"
+    }).output("/tmp/fairy_spec_testdata_multi.txt")
+      
+    result = File.readlines("/tmp/fairy_spec_testdata_multi.txt").inject({}){|res,ln|
+      ent = ln.split.map{|s| s.to_i}
+      res[ent[0]] = ent[1] 
+      res
+    }
+
+    result.should == answer
+  end
+end
+
+

data/spec/fairy6_spec.rb

@@ -0,0 +1,52 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source = File.readlines("testdata.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # sort_by
+  it 'should sort data' do
+    answer = @source.map{|ln|
+      n = ln.chomp.to_i
+      n
+    }.sort{|a,b|
+      a <=> b
+    }.map{|n|
+      "#{n}\n"
+    }
+
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp.to_i
+      n
+    }).sort_by(%{|n| n.to_i}).map(%q{|n|
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+                                  
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer
+
+    @fairy.input("testdata.vf").map(%{|ln|
+      n = ln.chomp.to_i
+      n
+    }).sort_by(%{|n| -(n.to_i)}).map(%q{|n|
+      "#{n}"
+    }).output("/tmp/fairy_spec_testdata.txt")
+    
+    result = File.readlines("/tmp/fairy_spec_testdata.txt")
+
+    result.should == answer.reverse
+  end
+end
+
+

data/spec/fairy7_spec.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    @source_join_a = File.readlines("testdata_join_a.txt")
+    @source_join_b = File.readlines("testdata_join_b.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # equijoin
+  it 'should do inner join' do
+    a = File.readlines("testdata_join_a.txt").map{|ln| rec = ln.split; rec}
+    b = File.readlines("testdata_join_b.txt").map{|ln| rec = ln.split; rec}
+
+    index = {}
+    b.each{|id,val|
+      index[id] ||= []
+      index[id] << val
+    }
+
+    answer = []
+    a.each{|id,val|
+      next unless index[id]
+      index[id].each{|val_b|
+        answer << [id, val, val_b]
+      }
+    }
+
+    answer = answer.sort_by{|ent| "%05d-%s-%s" % ent}.map{|ent|
+      ent.join("\t") + "\n"
+    }
+
+    a = @fairy.input("testdata_join_a.vf").map(%{|ln| rec = ln.split; rec})
+    b = @fairy.input("testdata_join_b.vf").map(%{|ln| rec = ln.split; rec})
+
+    joined = a.equijoin(b, 0)
+
+    joined.map(%{|from_a, from_b|
+      [from_a[0], from_a[1], from_b[1]]
+    }).sort_by(%{|ent| "%05d-%s-%s" % ent}).map(%{|ent|
+      ent.join("\t")
+    }).output("/tmp/fairy_spec_testdata_join.txt")
+
+    result = File.readlines("/tmp/fairy_spec_testdata_join.txt")
+
+    result.should == answer
+  end
+end
+
+

data/spec/fairy8_spec.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+# 
+# Copyright (C) 2007-2010 Rakuten, Inc.
+# 
+
+require 'rubygems'
+require 'rspec'
+require 'yaml'
+
+require 'fairy'
+
+describe Fairy do
+  before :all do
+    yml_path = File.expand_path(File.dirname(__FILE__) + "/../tools/cap_recipe/cluster.yml")
+    @cluster = YAML.load_file(yml_path)
+    @source = File.readlines("testdata.txt")
+    @fairy = Fairy::Fairy.new
+  end
+
+  # input (local) + split
+  it 'should split data & distribute them' do
+    answer = @source.sort
+
+    @fairy.input("testdata.txt").seg_split(10).output("/tmp/fairy_spec_testdata.vf")
+    system %{ fairy cat /tmp/fairy_spec_testdata.vf > /tmp/fairy_spec_testdata.txt }
+    result = File.readlines("/tmp/fairy_spec_testdata.txt").sort
+
+    result.should == answer
+  end
+
+  # exec
+  it 'should print all node-names' do
+    answer = @cluster["nodes"].sort
+    
+    result = []
+    @fairy.exec(@cluster["nodes"].map{|n| "file://#{n}"}).map(%q{|uri|
+      `hostname`.chomp
+    }).here.each{|n|
+      result << n
+    }
+
+    result.sort!
+
+    result.should == answer
+  end
+end
+
+