ruby-spark 1.1.0.1-java

data/spec/lib/reduce_by_key_spec.rb

@@ -0,0 +1,118 @@
+require "spec_helper"
+
+def flat_map(line)
+  line.split
+end
+
+def map(item)
+  [item, 1]
+end
+
+def reduce(x,y)
+  x+y
+end
+
+RSpec::shared_examples "a words counting" do |workers|
+  context "with #{workers || 'default'} worker" do
+    let(:result) do
+      keyyed = lines.flat_map{|x| x.split}.map{|x| [x,1]}
+      result = keyyed.reduce({}){|memo, item|
+        key   = item[0]
+        value = item[1]
+
+        memo[key] ||= 0
+        memo[key] += value
+        memo
+      }
+      result
+    end
+
+    it "when lambda" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(lambda{|line| line.split})
+      rdd2 = rdd2.map(lambda{|word| [word, 1]})
+      rdd2 = rdd2.reduce_by_key(lambda{|x,y| x+y})
+
+      expect(rdd2.collect_as_hash).to eql(result)
+    end
+
+    it "when method" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(method(:flat_map))
+      rdd2 = rdd2.map(method(:map))
+      rdd2 = rdd2.reduce_by_key(method(:reduce))
+
+      expect(rdd2.collect_as_hash).to eql(result)
+    end
+
+    it "keys, values" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(method(:flat_map))
+      rdd2 = rdd2.map(method(:map))
+      rdd2 = rdd2.reduce_by_key(method(:reduce))
+
+      expect(rdd2.keys.collect.sort).to eql(result.keys.sort)
+      expect { rdd2.values.collect.reduce(:+) }.to_not raise_error
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  context ".reduce_by_key" do
+    context "throught parallelize" do
+      let(:lines) { Generator.lines }
+
+      def rdd(workers)
+        $sc.parallelize(lines, workers)
+      end
+
+      it_behaves_like "a words counting", nil
+      it_behaves_like "a words counting", 1
+      it_behaves_like "a words counting", rand(2..10)
+    end
+
+    context "throught text_file" do
+      let(:file)  { File.join("spec", "inputs", "lorem_300.txt") }
+      let(:lines) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a words counting", nil
+      it_behaves_like "a words counting", 1
+      it_behaves_like "a words counting", rand(2..10)
+    end
+  end
+
+  context ".fold_by_key" do
+    let(:numbers)    { Generator.numbers }
+    let(:zero_value) { 0 }
+    let(:rdd)        { $sc.parallelize(numbers) }
+    let(:map)        { lambda{|x| [x, 1]} }
+    let(:add)        { lambda{|x,y| x+y} }
+
+    let(:result) do
+      _result = {}
+      numbers.map(&map).each do |key, value|
+        _result[key] ||= zero_value
+        _result[key] = add.call(_result[key], value)
+      end
+      _result
+    end
+
+    def fold_by_key(num_partitions=nil)
+      rdd.map(map).fold_by_key(zero_value, add, num_partitions).collect_as_hash
+    end
+
+    it "default num_partitions" do
+      expect(fold_by_key).to eq(result)
+    end
+
+    it "default num_partitions" do
+      expect(
+        fold_by_key rand(1..10)
+      ).to eq(result)
+    end
+  end
+end

data/spec/lib/reduce_spec.rb

@@ -0,0 +1,131 @@
+require "spec_helper"
+
+def longest_words(memo, word)
+  memo.length > word.length ? memo : word
+end
+
+RSpec::shared_examples "a reducing" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it ".reduce" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+      rdd2 = rdd2.reduce(func1)
+      result = numbers.map(&:to_i).reduce(&func1)
+
+      expect(rdd2).to eql(result)
+
+      rdd3 = rdd_numbers(workers)
+      rdd3 = rdd3.map(to_i)
+      rdd3 = rdd3.reduce(func2)
+      result = numbers.map(&:to_i).reduce(&func2)
+
+      expect(rdd3).to eql(result)
+
+      rdd4 = rdd_lines(workers)
+      rdd4 = rdd4.flat_map(split)
+      rdd4 = rdd4.reduce(method(:longest_words))
+
+      result = lines.flat_map(&split).reduce(&lambda(&method(:longest_words)))
+
+      expect(rdd4).to eql(result)
+    end
+
+    it ".fold" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+      rdd_result = rdd2.fold(1, func1)
+
+      # all workers add 1 + last reducing phase
+      result = numbers.map(&:to_i).reduce(&func1) + rdd2.partitions_size + 1
+
+      expect(rdd_result).to eql(result)
+    end
+
+    it ".aggregate" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      # Sum of items + their count
+      seq = lambda{|x,y| [x[0] + y, x[1] + 1]}
+      com = lambda{|x,y| [x[0] + y[0], x[1] + y[1]]}
+      rdd_result = rdd2.aggregate([0,0], seq, com)
+
+      result = [numbers.reduce(:+), numbers.size]
+
+      expect(rdd_result).to eql(result)
+    end
+
+    it ".max" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.max).to eql(numbers.map(&:to_i).max)
+    end
+
+    it ".min" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.min).to eql(numbers.map(&:to_i).min)
+    end
+
+    it ".sum" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.sum).to eql(numbers.map(&:to_i).reduce(:+))
+    end
+
+    it ".count" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.count).to eql(numbers.size)
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|sum, x| sum+x} }
+  let(:func2) { lambda{|product, x| product*x} }
+
+  let(:to_i)  { lambda{|item| item.to_i} }
+  let(:split) { lambda{|item| item.split} }
+
+  context "throught parallelize" do
+    let(:numbers) { Generator.numbers }
+    let(:lines)   { Generator.lines }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    def rdd_lines(workers)
+      $sc.parallelize(lines, workers)
+    end
+
+    it_behaves_like "a reducing", nil
+    it_behaves_like "a reducing", 1
+    it_behaves_like "a reducing", rand(2..10)
+  end
+
+  context "throught text_file" do
+    let(:file)       { File.join("spec", "inputs", "numbers_0_100.txt") }
+    let(:file_lines) { File.join("spec", "inputs", "lorem_300.txt") }
+
+    let(:numbers) { File.readlines(file).map(&:strip).map(&:to_i) }
+    let(:lines)   { File.readlines(file_lines).map(&:strip) }
+
+    def rdd_numbers(workers)
+      $sc.text_file(file, workers)
+    end
+
+    def rdd_lines(workers)
+      $sc.text_file(file_lines, workers)
+    end
+
+    it_behaves_like "a reducing", nil
+    it_behaves_like "a reducing", 1
+    it_behaves_like "a reducing", rand(2..10)
+  end
+end

data/spec/lib/sample_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+# Sample method can not be tested because of random generator
+# Just test it for raising error
+
+RSpec::shared_examples "a sampler" do |workers|
+  context "with #{workers || 'default'} worker" do
+
+    context ".sample" do
+      it "with replacement" do
+        rdd2 = rdd(workers).sample(true, rand)
+        expect { rdd2.collect }.to_not raise_error
+      end
+
+      it "without replacement" do
+        rdd2 = rdd(workers).sample(false, rand)
+        expect { rdd2.collect }.to_not raise_error
+      end
+    end
+
+    context ".take_sample" do
+      it "with replacement" do
+        size = rand(10..999)
+        expect(rdd(workers).take_sample(true, size).size).to eql(size)
+      end
+
+      it "without replacement" do
+        size = rand(10..999)
+        expect(rdd(workers).take_sample(false, size).size).to eql(size)
+      end
+    end
+
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:numbers) { Generator.numbers(1000) }
+
+  def rdd(workers)
+    $sc.parallelize(numbers, workers)
+  end
+
+  it_behaves_like "a sampler", nil
+  it_behaves_like "a sampler", 1
+  it_behaves_like "a sampler", rand(2..10)
+end

data/spec/lib/serializer_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+require 'zlib'
+
+RSpec.describe Spark::Serializer do
+  let(:data) { [1, 'test', 2.0, [3], {key: 'value'}, :test, String] }
+
+  it 'find' do
+    expect(described_class.find('not_existed_class')).to eql(nil)
+
+    expect(described_class.find('Marshal')).to eq(described_class::Marshal)
+    expect(described_class.find('marshal')).to eq(described_class::Marshal)
+    expect(described_class.find(:marshal)).to eq(described_class::Marshal)
+    expect(described_class.find('batched')).to eq(described_class::Batched)
+  end
+
+  it 'find!' do
+    expect { expect(described_class.find!('not_existed_class')) }.to raise_error(Spark::SerializeError)
+    expect { expect(described_class.find!('marshal')) }.to_not raise_error
+    expect { expect(described_class.find!('batched')) }.to_not raise_error
+  end
+
+  it 'register' do
+    NewSerializer = Class.new
+
+    expect(described_class.find('new_serializer_1')).to eql(nil)
+    expect(described_class.find('new_serializer_2')).to eql(nil)
+    expect(described_class.find('new_serializer_3')).to eql(nil)
+
+    described_class.register('new_serializer_1', 'new_serializer_2', 'new_serializer_3', NewSerializer)
+
+    expect(described_class.find('new_serializer_1')).to eql(NewSerializer)
+    expect(described_class.find('new_serializer_2')).to eql(NewSerializer)
+    expect(described_class.find('new_serializer_3')).to eql(NewSerializer)
+  end
+
+  it '==' do
+    # One class
+    marshal1 = described_class::Marshal.new
+    marshal2 = described_class::Marshal.new
+
+    expect(marshal1).to eq(marshal1)
+    expect(marshal1).to eq(marshal2)
+
+    # Two classes
+    compressed1 = described_class::Compressed.new(marshal1)
+    compressed2 = described_class::Compressed.new(marshal2)
+
+    expect(compressed1).to eq(compressed1)
+    expect(compressed1).to eq(compressed2)
+
+    # Three classes
+    batched1 = described_class::Batched.new(compressed1, 1)
+    batched2 = described_class::Batched.new(compressed2, 1)
+    batched3 = described_class::Batched.new(compressed1, 2)
+
+    expect(batched1).to eq(batched2)
+    expect(batched1).to_not eq(batched3)
+  end
+
+  context 'build' do
+    let(:marshal1)    { described_class::Marshal.new }
+    let(:compressed1) { described_class::Compressed.new(marshal1) }
+    let(:batched1)    { described_class::Batched.new(compressed1, 1) }
+
+    it 'block' do
+      expect(described_class.build{ marshal }).to eq(marshal1)
+      expect(described_class.build{ marshal }).to eq(described_class.build{ __marshal__ })
+      expect(described_class.build{ compressed(marshal) }).to eq(compressed1)
+      expect(described_class.build{ batched(compressed(marshal), 1) }).to eq(batched1)
+    end
+
+    it 'text' do
+      expect(described_class.build('marshal')).to eq(marshal1)
+      expect(described_class.build('compressed(marshal)')).to eq(compressed1)
+      expect(described_class.build('batched(compressed(marshal), 1)')).to eq(batched1)
+    end
+  end
+
+  it 'serialization' do
+    marshal1 = described_class.build{ marshal }
+    compressed1 = described_class.build{ compressed(marshal) }
+
+    expect(marshal1.dump(data)).to eq(Marshal.dump(data))
+    expect(compressed1.dump(data)).to eq(
+      Zlib::Deflate.deflate(Marshal.dump(data))
+    )
+  end
+end

data/spec/lib/sort_spec.rb

@@ -0,0 +1,58 @@
+require "spec_helper"
+
+RSpec::shared_examples "a sorting" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers)
+
+    rdd2 = rdd2.flat_map(split)
+    result = lines.flat_map(&split)
+
+    # Sort by self
+    rdd3 = rdd2.map(map).sort_by_key
+    result2 = result.map(&map).sort_by{|(key, _)| key}
+
+    expect(rdd3.collect).to eql(result2)
+
+    # Sort by len
+    rdd3 = rdd2.map(len_map).sort_by_key
+    result2 = result.map(&len_map).sort_by{|(key, _)| key}
+
+    expect(rdd3.collect).to eql(result2)
+  end
+end
+
+
+RSpec::describe "Spark::RDD" do
+  let(:split)   { lambda{|x| x.split} }
+  let(:map)     { lambda{|x| [x.to_s, 1]} }
+  let(:len_map) { lambda{|x| [x.size, x]} }
+
+  context "throught parallelize" do
+    context ".map" do
+      let(:lines) { Generator.lines }
+
+      def rdd(workers)
+        $sc.parallelize(lines, workers)
+      end
+
+      it_behaves_like "a sorting", nil
+      it_behaves_like "a sorting", 1
+      it_behaves_like "a sorting", rand(2..10)
+    end
+  end
+
+  context "throught text_file" do
+    context ".map" do
+      let(:file)  { File.join("spec", "inputs", "lorem_300.txt") }
+      let(:lines) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a sorting", nil
+      it_behaves_like "a sorting", 1
+      it_behaves_like "a sorting", rand(2..10)
+    end
+  end
+end

data/spec/lib/statistic_spec.rb

@@ -0,0 +1,170 @@
+require 'spec_helper'
+
+RSpec.shared_examples 'a stats' do |workers|
+  let(:numbers) { [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] }
+
+  context "with #{workers || 'default'} worker" do
+    it 'stats class' do
+      stats = $sc.parallelize(numbers, workers).stats
+
+      expect(stats.sum).to             be_within(0.1).of(20)
+      expect(stats.mean).to            be_within(0.1).of(20/6.0)
+      expect(stats.max).to             be_within(0.1).of(8.0)
+      expect(stats.min).to             be_within(0.1).of(1.0)
+      expect(stats.variance).to        be_within(0.1).of(6.22222)
+      expect(stats.sample_variance).to be_within(0.1).of(7.46667)
+      expect(stats.stdev).to           be_within(0.1).of(2.49444)
+      expect(stats.sample_stdev).to    be_within(0.1).of(2.73252)
+    end
+
+    it 'rdd methods' do
+      rdd = $sc.parallelize([1, 2, 3], workers)
+
+      expect(rdd.mean).to            be_within(0.1).of(2.0)
+      expect(rdd.variance).to        be_within(0.1).of(0.666)
+      expect(rdd.stdev).to           be_within(0.1).of(0.816)
+      expect(rdd.sample_stdev).to    be_within(0.1).of(1.0)
+      expect(rdd.sample_variance).to be_within(0.1).of(1.0)
+    end
+  end
+end
+
+RSpec.shared_examples 'a histogram' do |workers|
+
+  context "with #{workers || 'default'} worker" do
+    it 'empty' do
+      rdd = $sc.parallelize([], workers, ser)
+
+      expect( rdd.histogram([0, 10])[1] ).to eq([0])
+      expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
+    end
+
+    it 'validation' do
+      rdd = $sc.parallelize([], workers, ser)
+      expect { rdd.histogram(0) }.to raise_error(ArgumentError)
+    end
+
+    it 'double' do
+      rdd = $sc.parallelize([1.0, 2.0, 3.0, 4.0], workers, ser)
+      buckets, counts = rdd.histogram(2)
+
+      expect(buckets).to eq([1.0, 2.5, 4.0])
+      expect(counts).to eq([2, 2])
+    end
+
+    it 'out of range' do
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
+
+      expect( rdd.histogram([0, 10])[1] ).to eq([0])
+      expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
+    end
+
+    it 'in range with one bucket' do
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
+
+      expect( rdd.histogram([0, 10])[1] ).to eq([4])
+      expect( rdd.histogram([0, 4, 10])[1] ).to eq([3, 1])
+    end
+
+    it 'in range with one bucket exact match' do
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
+      expect( rdd.histogram([1, 4])[1] ).to eq([4])
+    end
+
+    it 'out of range with two buckets' do
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
+      expect( rdd.histogram([0, 5, 10])[1] ).to eq([0, 0])
+    end
+
+    it 'out of range with two uneven buckets' do
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
+      expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
+    end
+
+    it 'in range with two buckets' do
+      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, ser)
+      expect( rdd.histogram([0, 5, 10])[1] ).to eq([3, 2])
+    end
+
+    it 'in range with two bucket and nil' do
+      rdd = $sc.parallelize([1, 2, 3, 5, 6, nil, Float::NAN], workers, ser)
+      expect( rdd.histogram([0, 5, 10])[1] ).to eq([3, 2])
+    end
+
+    it 'in range with two uneven buckets' do
+      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, ser)
+      expect( rdd.histogram([0, 5, 11])[1] ).to eq([3, 2])
+    end
+
+    it 'mixed range with two uneven buckets' do
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01], workers, ser)
+      expect( rdd.histogram([0, 5, 11])[1] ).to eq([4, 3])
+    end
+
+    it 'mixed range with four uneven buckets' do
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1], workers, ser)
+      expect( rdd.histogram([0.0, 5.0, 11.0, 12.0, 200.0])[1] ).to eq([4, 2, 1, 3])
+    end
+
+    it 'mixed range with uneven buckets and NaN' do
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1, nil, Float::NAN], workers, ser)
+      expect( rdd.histogram([0.0, 5.0, 11.0, 12.0, 200.0])[1] ).to eq([4, 2, 1, 3])
+    end
+
+    it 'out of range with infinite buckets' do
+      rdd = $sc.parallelize([10.01, -0.01, Float::NAN, Float::INFINITY], workers, ser)
+      expect( rdd.histogram([-Float::INFINITY, 0, Float::INFINITY])[1] ).to eq([1, 1])
+    end
+
+    it 'without buckets' do
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
+      expect( rdd.histogram(1) ).to eq([[1, 4], [4]])
+    end
+
+    it 'without buckets single element' do
+      rdd = $sc.parallelize([1], workers, ser)
+      expect( rdd.histogram(1) ).to eq([[1, 1], [1]])
+    end
+
+    it 'without bucket no range' do
+      rdd = $sc.parallelize([1, 1, 1, 1], workers, ser)
+      expect( rdd.histogram(1) ).to eq([[1, 1], [4]])
+    end
+
+    it 'without buckets basic two' do
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
+      expect( rdd.histogram(2) ).to eq([[1, 2.5, 4], [2, 2]])
+    end
+
+    it 'without buckets with more requested than elements' do
+      rdd = $sc.parallelize([1, 2], workers, ser)
+      buckets = [1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0]
+      hist = [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]
+
+      expect( rdd.histogram(10) ).to eq([buckets, hist])
+    end
+
+    it 'string' do
+      rdd = $sc.parallelize(['ab', 'ac', 'b', 'bd', 'ef'], workers, ser)
+
+      expect( rdd.histogram(['a', 'b', 'c'])[1] ).to eq([2, 2])
+      expect( rdd.histogram(1) ).to eq([['ab', 'ef'], [5]])
+      expect { rdd.histogram(2) }.to raise_error(Spark::RDDError)
+    end
+
+  end
+end
+
+RSpec.describe Spark::RDD do
+  let(:ser) { Spark::Serializer.build { __batched__(__marshal__, 1) } }
+
+  context '.stats' do
+    it_behaves_like 'a stats', 1
+    it_behaves_like 'a stats', rand(2..5)
+  end
+
+  context '.histogram' do
+    it_behaves_like 'a histogram', 1
+    it_behaves_like 'a histogram', rand(2..5)
+  end
+end