ruby-spark 1.0.0

data/spec/lib/group_spec.rb

@@ -0,0 +1,109 @@
+require "spec_helper"
+
+RSpec::shared_examples "a groupping by key" do |workers|
+  it "with #{workers || 'default'} worker" do
+    expect(rdd_result(workers)).to eql(result)
+  end
+end
+
+RSpec::shared_examples "a cogroupping by key" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it ".group_with" do
+      rdd = rdd_1(workers).group_with(rdd_2(workers))
+      expect(rdd.collect_as_hash).to eql(result_12)
+    end
+
+    it ".cogroup" do
+      rdd = rdd_1(workers).cogroup(rdd_2(workers), rdd_3(workers))
+      expect(rdd.collect_as_hash).to eql(result_123)
+    end
+  end
+end
+
+RSpec::shared_examples "a groupping by" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd = rdd_numbers(workers)
+    rdd = rdd.group_by(key_function1)
+
+    expect(rdd.collect_as_hash).to eql(numbers.group_by(&key_function1))
+
+    rdd = rdd_words(workers)
+    rdd = rdd.group_by(key_function2)
+
+    expect(rdd.collect_as_hash).to eql(words.group_by(&key_function2))
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+
+  def make_result(*hashes)
+    _result = {}
+    hashes.each do |data|
+      data.each do |key, value|
+        _result[key] ||= []
+        _result[key] << value
+      end
+    end
+    _result
+  end
+
+  context ".group_by_key" do
+    let(:hash) { Generator.hash }
+    let(:result) { make_result(hash) }
+
+    def rdd_result(workers)
+      rdd = $sc.parallelize(hash)
+      rdd.group_by_key.collect_as_hash
+    end
+
+    it_behaves_like "a groupping by key", nil
+    it_behaves_like "a groupping by key", 1
+    it_behaves_like "a groupping by key", rand(2..10)
+  end
+
+  context "cogroup" do
+    let(:hash1) { Generator.hash }
+    let(:hash2) { Generator.hash }
+    let(:hash3) { Generator.hash }
+
+    let(:result_12)  { make_result(hash1, hash2) }
+    let(:result_123) { make_result(hash1, hash2, hash3) }
+
+    def rdd_1(workers)
+      $sc.parallelize(hash1)
+    end
+
+    def rdd_2(workers)
+      $sc.parallelize(hash2)
+    end
+
+    def rdd_3(workers)
+      $sc.parallelize(hash3)
+    end
+
+    it_behaves_like "a cogroupping by key", nil
+    it_behaves_like "a cogroupping by key", 1
+    it_behaves_like "a cogroupping by key", rand(2..10)
+  end
+
+  context "group_by" do
+    let(:key_function1) { lambda{|x| x%2} }
+    let(:key_function2) { lambda{|x| x.size} }
+
+    let(:numbers) { Generator.numbers }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words)
+    end
+
+    it_behaves_like "a groupping by", nil
+    it_behaves_like "a groupping by", 1
+    it_behaves_like "a groupping by", rand(2..10)
+  end
+
+end

data/spec/lib/helper_spec.rb

@@ -0,0 +1,19 @@
+require "spec_helper"
+
+RSpec::configure do |c|
+  c.include Spark::Helper::Parser
+end
+
+RSpec::describe Spark::Helper do
+  
+  it "memory size" do
+    expect(to_memory_size("512mb")).to eql(524288.0)
+    expect(to_memory_size("1586 mb")).to eql(1624064.0)
+    expect(to_memory_size("3 MB")).to eql(3072.0)
+    expect(to_memory_size("9gb")).to eql(9437184.0)
+    expect(to_memory_size("9gb", "mb")).to eql(9216.0)
+    expect(to_memory_size("9mb", "gb")).to eql(0.01)
+    expect(to_memory_size("6652548796kb", "mb")).to eql(6496629.68)
+  end
+
+end

data/spec/lib/key_spec.rb

@@ -0,0 +1,41 @@
+require "spec_helper"
+
+RSpec::shared_examples "a keying by" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd = rdd_numbers(workers)
+    rdd = rdd.key_by(key_function1)
+
+    result = numbers.map{|item| [key_function1.call(item), item]}
+    expect(rdd.collect).to eql(result)
+
+    rdd = rdd_words(workers)
+    rdd = rdd.key_by(key_function2)
+
+    result = words.map{|item| [key_function2.call(item), item]}
+    expect(rdd.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+
+  context "key_by" do
+    let(:key_function1) { lambda{|x| x.even?} }
+    let(:key_function2) { lambda{|x| x.include?("a")} }
+
+    let(:numbers) { Generator.numbers }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words)
+    end
+
+    it_behaves_like "a keying by", nil
+    it_behaves_like "a keying by", 1
+    it_behaves_like "a keying by", rand(2..10)
+  end
+
+end

data/spec/lib/manipulation_spec.rb

@@ -0,0 +1,114 @@
+require "spec_helper"
+
+RSpec::describe "Spark::RDD" do
+  let(:numbers) { 1..100 }
+  let(:rand_numbers) { Generator.numbers }
+
+  it ".glom" do
+    rdd = $sc.parallelize(numbers, 1).glom
+    expect(rdd.collect).to eql([numbers.to_a])
+
+    rdd = $sc.parallelize(numbers, 5, batch_size: 1).glom
+    expect(rdd.collect).to eql(numbers.each_slice(20).to_a)
+  end
+
+  it ".coalesce" do
+    rdd = $sc.parallelize(numbers, 5)
+
+    rdd2 = rdd.glom
+    expect(rdd2.collect.size).to eql(5)
+
+    rdd3 = rdd.coalesce(4).glom
+    expect(rdd3.collect.size).to eql(4)
+  end
+
+  it ".distinct" do
+    rdd = $sc.parallelize(rand_numbers, 5)
+    rdd = rdd.distinct
+    expect(rdd.collect.sort).to eql(rand_numbers.uniq.sort)
+
+    rdd = $sc.parallelize(numbers, 5)
+    rdd = rdd.map(lambda{|x| 1})
+    rdd = rdd.distinct
+    expect(rdd.collect).to eql([1])
+  end
+
+  context ".union" do
+    it "classic method" do
+      rdd = $sc.parallelize(numbers, 5)
+      rdd = rdd.union(rdd).collect
+
+      expect(rdd.collect.sort).to eql((numbers.to_a+numbers.to_a).sort)
+    end
+
+    it "with a different serializer" do
+      rdd1 = $sc.parallelize(numbers, 1, serializer: "marshal")
+      rdd2 = $sc.parallelize(numbers, 1, serializer: "oj")
+      
+      expect { rdd1.union(rdd2).collect }.to_not raise_error
+    end
+
+    it "as operator" do
+      rdd1 = $sc.parallelize(numbers)
+      rdd2 = $sc.parallelize(rand_numbers)
+
+      expect((rdd1+rdd2).sum).to eql((numbers.to_a+rand_numbers).reduce(:+))
+    end
+  end
+
+  it ".compact" do
+    data = [nil, nil , 0, 0, 1, 2, nil, 6]
+    result = data.compact
+
+    rdd = $sc.parallelize(data, 1).compact
+    expect(rdd.collect).to eql(result)
+
+    rdd = $sc.parallelize(data, 5, batch_size: 1).compact
+    expect(rdd.collect).to eql(result)
+
+    rdd = $sc.parallelize(data, 1, batch_size: 1).compact
+    expect(rdd.collect).to eql(result)
+  end
+
+  it ".intersection" do
+    data1 = [0,1,2,3,4,5,6,7,8,9,10]
+    data2 = [5,6,7,8,9,10,11,12,13,14,15]
+
+    rdd1 = $sc.parallelize(data1)
+    rdd2 = $sc.parallelize(data2)
+
+    expect(rdd1.intersection(rdd2).collect.sort).to eql(data1 & data2)
+  end
+
+  it ".shuffle" do
+    data = Generator.numbers
+    rdd = $sc.parallelize(data)
+
+    expect(rdd.shuffle.collect).to_not eql(data)
+  end
+
+  context ".cartesian" do
+    let(:data1) { Generator.numbers(100) }
+    let(:data2) { Generator.numbers(100) }
+    let(:result) { data1.product(data2).map(&:to_s).sort }
+
+    it "unbatched" do
+      rdd1 = $sc.parallelize(data1, 2, batch_size: 1)
+      rdd2 = $sc.parallelize(data2, 2, batch_size: 1)
+
+      rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
+
+      expect(rdd.collect.sort).to eql(result)
+    end
+
+    it "batched" do
+      rdd1 = $sc.parallelize(data1, 2, batch_size: rand(4..10))
+      rdd2 = $sc.parallelize(data2, 2, batch_size: rand(4..10))
+
+      rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
+
+      expect(rdd.collect.sort).to eql(result)
+    end
+  end
+
+end

data/spec/lib/map_partitions_spec.rb

@@ -0,0 +1,87 @@
+require "spec_helper"
+
+def func3(x)
+  x.map(&:to_i).reduce(:+)
+end
+
+def func4_with_index(data, index)
+  {
+    index => data.map(&:to_i).reduce(:*)
+  }
+end
+
+RSpec::shared_examples "a map partitions" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it "without index" do
+      rdd2 = rdd(workers).map_partitions(func1)
+      result = func1.call(numbers)
+
+      expect(func1.call(rdd2.collect)).to eql(result)
+
+      rdd3 = rdd(workers)
+      rdd3 = rdd3.map_partitions(func1)
+      rdd3 = rdd3.map_partitions(func2)
+      rdd3 = rdd3.map_partitions(method(:func3))
+      result = func3(func2.call(func1.call(numbers)))
+
+      # Not same number of workers
+      expect(rdd3.collect.size).to be >= 1
+
+      rdd4 = rdd(workers)
+      rdd4 = rdd4.map_partitions(func1)
+      rdd4 = rdd4.map_partitions(func2)
+      rdd4 = rdd4.map_partitions(method(:func3))
+
+      expect(rdd4.collect).to eql(rdd3.collect)
+    end
+
+    it "with index" do
+      rdd2 = rdd(workers).map_partitions_with_index(method(:func4_with_index))
+      result = rdd2.collect
+
+      expect(result).to be_a(Array)
+
+      result.each do |x|
+        expect(x).to be_a(Hash)
+      end
+
+      # Multiply by 0
+      # Some values are 0 because of batched serialization
+      expect(result.map(&:values).flatten.compact.uniq.first).to eql(0)
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD.map_partitions(_with_index)" do
+  let(:func1) { lambda{|x| x.map(&:to_i)} }
+  let(:func2) {
+    lambda{|x|
+      x.map{|y| y*2}
+    }
+  }
+
+  context "throught parallelize" do
+    let(:numbers) { 0..1000 }
+
+    def rdd(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    it_behaves_like "a map partitions", nil
+    it_behaves_like "a map partitions", 1
+    it_behaves_like "a map partitions", rand(2..10)
+  end
+
+  context "throught text_file" do
+    let(:file)    { File.join("spec", "inputs", "numbers_0_100.txt") }
+    let(:numbers) { File.readlines(file).map(&:strip) }
+
+    def rdd(workers)
+      $sc.text_file(file, workers)
+    end
+
+    it_behaves_like "a map partitions", nil
+    it_behaves_like "a map partitions", 1
+    it_behaves_like "a map partitions", rand(2..10)
+  end
+end

data/spec/lib/map_spec.rb

@@ -0,0 +1,91 @@
+require "spec_helper"
+
+RSpec::shared_examples "a mapping" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map(func1)
+    result = numbers.map(&func1)
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.map(func1)
+    rdd3 = rdd3.map(func2)
+    rdd3 = rdd3.map(func3)
+    result = numbers.map(&func1).map(&func2).map(&func3)
+
+    expect(rdd3.collect).to eql(result)
+
+    rdd4 = rdd(workers)
+    rdd4 = rdd4.map(func3)
+    rdd4 = rdd4.map(func2)
+    rdd4 = rdd4.map(func1)
+
+    expect(rdd4.collect).to eql(rdd3.collect)
+  end
+end
+
+RSpec::shared_examples "a mapping values" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map_values(func1)
+    result = hash.map{|key, value| [key, func1.call(value)]}
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.map_values(func1)
+    rdd3 = rdd3.map_values(func2)
+    rdd3 = rdd3.map_values(func3)
+    result = hash.map{|key, value| [key, func1.call(value)]}
+                 .map{|key, value| [key, func2.call(value)]}
+                 .map{|key, value| [key, func3.call(value)]}
+
+    expect(rdd3.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|x| x*2} }
+  let(:func2) { lambda{|x| x*3} }
+  let(:func3) { lambda{|x| x*4} }
+
+  context "throught parallelize" do
+    context ".map" do
+      let(:numbers) { Generator.numbers }
+
+      def rdd(workers)
+        $sc.parallelize(numbers, workers)
+      end
+
+      it_behaves_like "a mapping", nil
+      it_behaves_like "a mapping", 1
+      it_behaves_like "a mapping", rand(2..10)
+    end
+
+    context ".map_values" do
+      let!(:hash) { Generator.hash }
+
+      def rdd(workers)
+        $sc.parallelize(hash, workers)
+      end
+
+      it_behaves_like "a mapping values", nil
+      it_behaves_like "a mapping values", 1
+      it_behaves_like "a mapping values", rand(2..10)
+    end
+  end
+
+  context "throught text_file" do
+    context ".map" do
+      let(:file) { File.join("spec", "inputs", "numbers_0_100.txt") }
+      let(:numbers) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a mapping", nil
+      it_behaves_like "a mapping", 1
+      it_behaves_like "a mapping", rand(2..10)
+    end
+  end
+end