ruby-spark 1.1.0.1-java

data/spec/lib/external_apps_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+RSpec::describe Spark::RDD do
+
+  context '.pipe' do
+    let(:words)   { Generator.words }
+    let(:numbers) { Generator.numbers }
+
+    it 'single program' do
+      skip if windows?
+
+      rdd = $sc.parallelize(words, 1)
+      rdd = rdd.pipe('tr a b')
+
+      result = words.dup
+      result.map! do |x|
+        x.gsub('a', 'b')
+      end
+
+      expect(rdd.collect).to eql(result)
+    end
+
+    it 'multiple program' do
+      skip if windows?
+
+      rdd = $sc.parallelize(numbers, 1)
+      rdd = rdd.pipe("tr 1 5", "awk '{print $1*10}'")
+      rdd = rdd.map(lambda{|x| x.to_i * 100})
+
+      result = numbers.dup
+      result.map! do |x|
+        x.to_s.gsub('1', '5')
+      end
+      result.map! do |x|
+        x.to_i * 10
+      end
+      result.map! do |x|
+        x * 100
+      end
+
+      expect(rdd.collect).to eql(result)
+    end
+  end
+
+end

data/spec/lib/filter_spec.rb

@@ -0,0 +1,80 @@
+require "spec_helper"
+
+def func4(item)
+  item.start_with?("a") && item.size > 3 && item[1].to_s.ord > 106
+end
+
+RSpec::shared_examples "a filtering" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it "when numbers" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.filter(func1)
+      result = numbers.select(&func1)
+
+      expect(rdd2.collect).to eql(result)
+
+      rdd3 = rdd_numbers(workers)
+      rdd3 = rdd3.filter(func1)
+      rdd3 = rdd3.filter(func2)
+
+      expect(rdd3.collect).to eql([])
+    end
+
+    it "when words" do
+      rdd2 = rdd_words(workers)
+      rdd2 = rdd2.filter(func3)
+      result = words.select{|x| func3.call(x)}
+
+      expect(rdd2.collect).to eql(result)
+
+      rdd3 = rdd_words(workers)
+      rdd3 = rdd3.filter(method(:func4))
+      result = words.select{|x| func4(x)}
+
+      expect(rdd3.collect).to eql(result)
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD.filter" do
+  let(:func1) { lambda{|x| x.to_i.even?} }
+  let(:func2) { lambda{|x| x.to_i.odd?} }
+  let(:func3) { lambda{|x| x.to_s.start_with?("b")} }
+
+  context "throught parallelize" do
+    let(:numbers) { Generator.numbers_with_zero }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words, workers)
+    end
+
+    it_behaves_like "a filtering", nil
+    it_behaves_like "a filtering", 1
+    it_behaves_like "a filtering", rand(2..10)
+  end
+
+  context "throught text_file" do
+    let(:file_numbers) { File.join("spec", "inputs", "numbers_0_100.txt") }
+    let(:file_words)   { File.join("spec", "inputs", "lorem_300.txt") }
+
+    let(:numbers) { File.readlines(file_numbers).map(&:strip) }
+    let(:words)   { File.readlines(file_words).map(&:strip) }
+
+    def rdd_numbers(workers)
+      $sc.text_file(file_numbers, workers)
+    end
+
+    def rdd_words(workers)
+      $sc.text_file(file_words, workers)
+    end
+
+    it_behaves_like "a filtering", nil
+    it_behaves_like "a filtering", 1
+    it_behaves_like "a filtering", rand(2..10)
+  end
+end

data/spec/lib/flat_map_spec.rb

@@ -0,0 +1,100 @@
+require "spec_helper"
+
+RSpec::shared_examples "a flat mapping" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map(func1)
+    result = numbers.flat_map(&func1)
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.flat_map(func1)
+    rdd3 = rdd3.flat_map(func2)
+    rdd3 = rdd3.flat_map(func3)
+    result = numbers.flat_map(&func1).flat_map(&func2).flat_map(&func3)
+
+    expect(rdd3.collect).to eql(result)
+
+    rdd4 = rdd(workers)
+    rdd4 = rdd4.flat_map(func1)
+    rdd4 = rdd4.flat_map(func2)
+    rdd4 = rdd4.flat_map(func3)
+
+    expect(rdd4.collect).to eql(rdd3.collect)
+  end
+end
+
+RSpec::shared_examples "a flat mapping values" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).flat_map_values(func1)
+    result = []
+    hash_with_values.each do |(key, values)|
+      values = func1.call(values).flatten
+      values.each do |value|
+        result << [key, value]
+      end
+    end
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd2 = rdd(workers).flat_map_values(func2)
+    result = []
+    hash_with_values.each do |(key, values)|
+      values = func2.call(values).flatten
+      values.each do |value|
+        result << [key, value]
+      end
+    end
+
+    expect(rdd2.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|x| x*2} }
+  let(:func2) { lambda{|x| [x*3, 1, 1]} }
+  let(:func3) { lambda{|x| [x*4, 2, 2]} }
+
+  context "throught parallelize" do
+    context ".flat_map" do
+      let(:numbers) { Generator.numbers_with_zero }
+
+      def rdd(workers)
+        $sc.parallelize(numbers, workers)
+      end
+
+      it_behaves_like "a flat mapping", nil
+      it_behaves_like "a flat mapping", 1
+      it_behaves_like "a flat mapping", rand(2..10)
+    end
+
+    context ".flat_map_values" do
+      let(:func1) { lambda{|x| x*2} }
+      let(:func2) { lambda{|x| [x.first]} }
+      let(:hash_with_values) { Generator.hash_with_values }
+
+      def rdd(workers)
+        $sc.parallelize(hash_with_values, workers)
+      end
+
+      it_behaves_like "a flat mapping values", nil
+      it_behaves_like "a flat mapping values", 1
+      it_behaves_like "a flat mapping values", rand(2..10)
+    end
+  end
+
+  context "throught text_file" do
+    context ".flat_map" do
+      let(:file)    { File.join("spec", "inputs", "numbers_0_100.txt") }
+      let(:numbers) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a flat mapping", nil
+      it_behaves_like "a flat mapping", 1
+      it_behaves_like "a flat mapping", rand(2..10)
+    end
+  end
+end

data/spec/lib/group_spec.rb

@@ -0,0 +1,109 @@
+require "spec_helper"
+
+RSpec::shared_examples "a groupping by key" do |workers|
+  it "with #{workers || 'default'} worker" do
+    expect(rdd_result(workers)).to eql(result)
+  end
+end
+
+RSpec::shared_examples "a cogroupping by key" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it ".group_with" do
+      rdd = rdd_1(workers).group_with(rdd_2(workers))
+      expect(rdd.collect_as_hash).to eql(result_12)
+    end
+
+    it ".cogroup" do
+      rdd = rdd_1(workers).cogroup(rdd_2(workers), rdd_3(workers))
+      expect(rdd.collect_as_hash).to eql(result_123)
+    end
+  end
+end
+
+RSpec::shared_examples "a groupping by" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd = rdd_numbers(workers)
+    rdd = rdd.group_by(key_function1)
+
+    expect(rdd.collect_as_hash).to eql(numbers.group_by(&key_function1))
+
+    rdd = rdd_words(workers)
+    rdd = rdd.group_by(key_function2)
+
+    expect(rdd.collect_as_hash).to eql(words.group_by(&key_function2))
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+
+  def make_result(*hashes)
+    _result = {}
+    hashes.each do |data|
+      data.each do |key, value|
+        _result[key] ||= []
+        _result[key] << value
+      end
+    end
+    _result
+  end
+
+  context ".group_by_key" do
+    let(:hash) { Generator.hash }
+    let(:result) { make_result(hash) }
+
+    def rdd_result(workers)
+      rdd = $sc.parallelize(hash)
+      rdd.group_by_key.collect_as_hash
+    end
+
+    it_behaves_like "a groupping by key", nil
+    it_behaves_like "a groupping by key", 1
+    it_behaves_like "a groupping by key", rand(2..10)
+  end
+
+  context "cogroup" do
+    let(:hash1) { Generator.hash }
+    let(:hash2) { Generator.hash }
+    let(:hash3) { Generator.hash }
+
+    let(:result_12)  { make_result(hash1, hash2) }
+    let(:result_123) { make_result(hash1, hash2, hash3) }
+
+    def rdd_1(workers)
+      $sc.parallelize(hash1)
+    end
+
+    def rdd_2(workers)
+      $sc.parallelize(hash2)
+    end
+
+    def rdd_3(workers)
+      $sc.parallelize(hash3)
+    end
+
+    it_behaves_like "a cogroupping by key", nil
+    it_behaves_like "a cogroupping by key", 1
+    it_behaves_like "a cogroupping by key", rand(2..10)
+  end
+
+  context "group_by" do
+    let(:key_function1) { lambda{|x| x%2} }
+    let(:key_function2) { lambda{|x| x.size} }
+
+    let(:numbers) { Generator.numbers }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words)
+    end
+
+    it_behaves_like "a groupping by", nil
+    it_behaves_like "a groupping by", 1
+    it_behaves_like "a groupping by", rand(2..10)
+  end
+
+end

data/spec/lib/helper_spec.rb

@@ -0,0 +1,19 @@
+require "spec_helper"
+
+RSpec::configure do |c|
+  c.include Spark::Helper::Parser
+end
+
+RSpec::describe Spark::Helper do
+  
+  it "memory size" do
+    expect(to_memory_size("512mb")).to eql(524288.0)
+    expect(to_memory_size("1586 mb")).to eql(1624064.0)
+    expect(to_memory_size("3 MB")).to eql(3072.0)
+    expect(to_memory_size("9gb")).to eql(9437184.0)
+    expect(to_memory_size("9gb", "mb")).to eql(9216.0)
+    expect(to_memory_size("9mb", "gb")).to eql(0.01)
+    expect(to_memory_size("6652548796kb", "mb")).to eql(6496629.68)
+  end
+
+end

data/spec/lib/key_spec.rb

@@ -0,0 +1,41 @@
+require "spec_helper"
+
+RSpec::shared_examples "a keying by" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd = rdd_numbers(workers)
+    rdd = rdd.key_by(key_function1)
+
+    result = numbers.map{|item| [key_function1.call(item), item]}
+    expect(rdd.collect).to eql(result)
+
+    rdd = rdd_words(workers)
+    rdd = rdd.key_by(key_function2)
+
+    result = words.map{|item| [key_function2.call(item), item]}
+    expect(rdd.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+
+  context "key_by" do
+    let(:key_function1) { lambda{|x| x.even?} }
+    let(:key_function2) { lambda{|x| x.include?("a")} }
+
+    let(:numbers) { Generator.numbers }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words)
+    end
+
+    it_behaves_like "a keying by", nil
+    it_behaves_like "a keying by", 1
+    it_behaves_like "a keying by", rand(2..10)
+  end
+
+end

data/spec/lib/manipulation_spec.rb

@@ -0,0 +1,122 @@
+require "spec_helper"
+
+RSpec::describe "Spark::RDD" do
+  let(:numbers) { 1..100 }
+  let(:rand_numbers) { Generator.numbers }
+
+  it ".glom" do
+    rdd = $sc.parallelize(numbers, 1).glom
+    expect(rdd.collect).to eql([numbers.to_a])
+
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+    rdd = $sc.parallelize(numbers, 5, ser).glom
+    expect(rdd.collect).to eql(numbers.each_slice(20).to_a)
+  end
+
+  it ".coalesce" do
+    rdd = $sc.parallelize(numbers, 5)
+
+    rdd2 = rdd.glom
+    expect(rdd2.collect.size).to eql(5)
+
+    rdd3 = rdd.coalesce(4).glom
+    expect(rdd3.collect.size).to eql(4)
+  end
+
+  it ".distinct" do
+    rdd = $sc.parallelize(rand_numbers, 5)
+    rdd = rdd.distinct
+    expect(rdd.collect.sort).to eql(rand_numbers.uniq.sort)
+
+    rdd = $sc.parallelize(numbers, 5)
+    rdd = rdd.map(lambda{|x| 1})
+    rdd = rdd.distinct
+    expect(rdd.collect).to eql([1])
+  end
+
+  context ".union" do
+    it "classic method" do
+      rdd = $sc.parallelize(numbers, 5)
+      rdd = rdd.union(rdd).collect
+
+      expect(rdd.collect.sort).to eql((numbers.to_a+numbers.to_a).sort)
+    end
+
+    it "with a different serializer" do
+      rdd1 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__marshal__) })
+      rdd2 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__oj__) })
+
+      expect { rdd1.union(rdd2).collect }.to_not raise_error
+    end
+
+    it "as operator" do
+      rdd1 = $sc.parallelize(numbers)
+      rdd2 = $sc.parallelize(rand_numbers)
+
+      expect((rdd1+rdd2).sum).to eql((numbers.to_a+rand_numbers).reduce(:+))
+    end
+  end
+
+  it ".compact" do
+    data = [nil, nil , 0, 0, 1, 2, nil, 6]
+    result = data.compact
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+    rdd = $sc.parallelize(data, 1).compact
+    expect(rdd.collect).to eql(result)
+
+    rdd = $sc.parallelize(data, 5, ser).compact
+    expect(rdd.collect).to eql(result)
+
+    rdd = $sc.parallelize(data, 1, ser).compact
+    expect(rdd.collect).to eql(result)
+  end
+
+  it ".intersection" do
+    data1 = [0,1,2,3,4,5,6,7,8,9,10]
+    data2 = [5,6,7,8,9,10,11,12,13,14,15]
+
+    rdd1 = $sc.parallelize(data1)
+    rdd2 = $sc.parallelize(data2)
+
+    expect(rdd1.intersection(rdd2).collect.sort).to eql(data1 & data2)
+  end
+
+  it ".shuffle" do
+    data = Generator.numbers
+    rdd = $sc.parallelize(data)
+
+    expect(rdd.shuffle.collect).to_not eql(data)
+  end
+
+  context ".cartesian" do
+    let(:data1) { Generator.numbers(100) }
+    let(:data2) { Generator.numbers(100) }
+    let(:result) { data1.product(data2).map(&:to_s).sort }
+
+    it "unbatched" do
+      ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+      rdd1 = $sc.parallelize(data1, 2, ser)
+      rdd2 = $sc.parallelize(data2, 2, ser)
+
+      rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
+
+      expect(rdd.collect.sort).to eql(result)
+    end
+
+    it "batched" do
+      ser1 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
+      ser2 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
+
+      rdd1 = $sc.parallelize(data1, 2, ser1)
+      rdd2 = $sc.parallelize(data2, 2, ser2)
+
+      rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
+
+      expect(rdd.collect.sort).to eql(result)
+    end
+  end
+
+end