ruby-spark 1.1.0.1-java

data/spec/lib/map_partitions_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+def func3(x)
+  x.map(&:to_i).reduce(:+)
+end
+
+def func4_with_index(data, index)
+  [{
+    index => data.map(&:to_i).reduce(:*)
+  }]
+end
+
+RSpec.shared_examples 'a map partitions' do |workers|
+  context "with #{workers || 'default'} worker" do
+    it 'without index' do
+      rdd2 = rdd(workers).map_partitions(func1)
+      result = func1.call(numbers)
+
+      expect(func1.call(rdd2.collect)).to eql(result)
+
+      rdd3 = rdd(workers)
+      rdd3 = rdd3.map_partitions(func1)
+      rdd3 = rdd3.map_partitions(func2)
+      rdd3 = rdd3.map_partitions(method(:func3))
+      result = func3(func2.call(func1.call(numbers)))
+
+      # Not same number of workers
+      expect(rdd3.collect.size).to be >= 1
+
+      rdd4 = rdd(workers)
+      rdd4 = rdd4.map_partitions(func1)
+      rdd4 = rdd4.map_partitions(func2)
+      rdd4 = rdd4.map_partitions(method(:func3))
+
+      expect(rdd4.collect).to eql(rdd3.collect)
+    end
+
+    it 'with index' do
+      rdd2 = rdd(workers).map_partitions_with_index(method(:func4_with_index))
+      result = rdd2.collect
+
+      expect(result).to be_a(Array)
+
+      result.each do |x|
+        expect(x).to be_a(Hash)
+      end
+
+      # Multiply by 0
+      # Some values are 0 because of batched serialization
+      expect(result.map(&:values).flatten.compact.uniq.first).to eql(0)
+    end
+  end
+end
+
+RSpec::describe 'Spark::RDD.map_partitions(_with_index)' do
+  let(:func1) { lambda{|x| x.map(&:to_i)} }
+  let(:func2) {
+    lambda{|x|
+      x.map{|y| y*2}
+    }
+  }
+
+  context 'throught parallelize' do
+    let(:numbers) { 0..1000 }
+
+    def rdd(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    it_behaves_like 'a map partitions', nil
+    it_behaves_like 'a map partitions', 1
+    it_behaves_like 'a map partitions', rand(2..10)
+  end
+
+  context 'throught text_file' do
+    let(:file)    { File.join('spec', 'inputs', 'numbers_0_100.txt') }
+    let(:numbers) { File.readlines(file).map(&:strip) }
+
+    def rdd(workers)
+      $sc.text_file(file, workers)
+    end
+
+    it_behaves_like 'a map partitions', nil
+    it_behaves_like 'a map partitions', 1
+    it_behaves_like 'a map partitions', rand(2..10)
+  end
+end

data/spec/lib/map_spec.rb

@@ -0,0 +1,91 @@
+require "spec_helper"
+
+RSpec::shared_examples "a mapping" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map(func1)
+    result = numbers.map(&func1)
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.map(func1)
+    rdd3 = rdd3.map(func2)
+    rdd3 = rdd3.map(func3)
+    result = numbers.map(&func1).map(&func2).map(&func3)
+
+    expect(rdd3.collect).to eql(result)
+
+    rdd4 = rdd(workers)
+    rdd4 = rdd4.map(func3)
+    rdd4 = rdd4.map(func2)
+    rdd4 = rdd4.map(func1)
+
+    expect(rdd4.collect).to eql(rdd3.collect)
+  end
+end
+
+RSpec::shared_examples "a mapping values" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map_values(func1)
+    result = hash.map{|key, value| [key, func1.call(value)]}
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.map_values(func1)
+    rdd3 = rdd3.map_values(func2)
+    rdd3 = rdd3.map_values(func3)
+    result = hash.map{|key, value| [key, func1.call(value)]}
+                 .map{|key, value| [key, func2.call(value)]}
+                 .map{|key, value| [key, func3.call(value)]}
+
+    expect(rdd3.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|x| x*2} }
+  let(:func2) { lambda{|x| x*3} }
+  let(:func3) { lambda{|x| x*4} }
+
+  context "throught parallelize" do
+    context ".map" do
+      let(:numbers) { Generator.numbers }
+
+      def rdd(workers)
+        $sc.parallelize(numbers, workers)
+      end
+
+      it_behaves_like "a mapping", nil
+      it_behaves_like "a mapping", 1
+      it_behaves_like "a mapping", rand(2..10)
+    end
+
+    context ".map_values" do
+      let!(:hash) { Generator.hash }
+
+      def rdd(workers)
+        $sc.parallelize(hash, workers)
+      end
+
+      it_behaves_like "a mapping values", nil
+      it_behaves_like "a mapping values", 1
+      it_behaves_like "a mapping values", rand(2..10)
+    end
+  end
+
+  context "throught text_file" do
+    context ".map" do
+      let(:file) { File.join("spec", "inputs", "numbers_0_100.txt") }
+      let(:numbers) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a mapping", nil
+      it_behaves_like "a mapping", 1
+      it_behaves_like "a mapping", rand(2..10)
+    end
+  end
+end

data/spec/lib/mllib/classification_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib classification' do
+
+  let(:data1) do
+    [
+      LabeledPoint.new(0.0, [1, 0, 0]),
+      LabeledPoint.new(1.0, [0, 1, 1]),
+      LabeledPoint.new(0.0, [2, 0, 0]),
+      LabeledPoint.new(1.0, [0, 2, 1])
+    ]
+  end
+
+  let(:values1) do
+    data1.map do |lp|
+      lp.features.values
+    end
+  end
+
+  let(:rdd1) { $sc.parallelize(data1) }
+
+  context 'logistic regression' do
+    it 'test' do
+      lrm = LogisticRegressionWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+
+  context 'svm' do
+    it 'test' do
+      lrm = SVMWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+
+  context 'naive bayes' do
+    it 'test' do
+      lrm = NaiveBayes.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+end

data/spec/lib/mllib/clustering_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib clustering' do
+  context 'kmeans' do
+    it 'test' do
+      data = [
+        DenseVector.new([0, 1.1]),
+        DenseVector.new([0, 1.2]),
+        DenseVector.new([1.1, 0]),
+        DenseVector.new([1.2, 0])
+      ]
+      model = KMeans.train($sc.parallelize(data), 2, initialization_mode: 'k-means||')
+
+      expect(model.predict(data[0])).to eq(model.predict(data[1]))
+      expect(model.predict(data[2])).to eq(model.predict(data[3]))
+    end
+
+    it 'deterministic' do
+      data = Array.new(10) do |i|
+        i *= 10
+        DenseVector.new([i, i])
+      end
+
+      clusters1 = KMeans.train($sc.parallelize(data), 3, initialization_mode: 'k-means||', seed: 42)
+      clusters2 = KMeans.train($sc.parallelize(data), 3, initialization_mode: 'k-means||', seed: 42)
+
+      centers1 = clusters1.centers.to_a
+      centers2 = clusters2.centers.to_a
+
+      centers1.zip(centers2).each do |c1, c2|
+        expect(c1).to eq(c2)
+      end
+    end
+  end
+end

data/spec/lib/mllib/matrix_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib::Matrix' do
+  context 'dense' do
+    it 'construct' do
+      values = [[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]]
+      matrix = DenseMatrix.new(3, 3, [[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]])
+
+      expect(matrix.shape).to eq([3, 3])
+      expect(matrix.values).to eq([[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]])
+    end
+  end
+
+  context 'sparse' do
+    it 'construct' do
+      values = [1.0, 2.0, 4.0, 5.0]
+      column_pointers = [0, 2, 2, 4, 4]
+      row_indices = [1, 2, 1, 2]
+
+      matrix = SparseMatrix.new(3, 4, column_pointers, row_indices, values)
+
+      expect(matrix.shape).to eq([3, 4])
+      expect(matrix.to_a).to eq(
+        [
+          [0.0, 0.0, 0.0, 0.0],
+          [1.0, 0.0, 4.0, 0.0],
+          [2.0, 0.0, 5.0, 0.0]
+        ]
+      )
+    end
+  end
+end

data/spec/lib/mllib/regression_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+
+# Mllib functions are tested on Spark
+# This just test if ruby call proper methods
+
+RSpec.describe 'Spark::Mllib regression' do
+
+  let(:data1) do
+    [
+      LabeledPoint.new(-1.0, [0, -1]),
+      LabeledPoint.new(1.0, [0, 1]),
+      LabeledPoint.new(-1.0, [0, -2]),
+      LabeledPoint.new(1.0, [0, 2])
+    ]
+  end
+
+  let(:values1) do
+    data1.map do |lp|
+      lp.features.values
+    end
+  end
+
+  let(:rdd1) { $sc.parallelize(data1) }
+
+  context 'labeled point' do
+    let(:lp) { LabeledPoint.new(1, [1,2,3]) }
+
+    it 'from array' do
+      expect(lp.label).to eql(1.0)
+      expect(lp.features).to be_a(DenseVector)
+    end
+
+    it 'serialize' do
+      lp2 = Marshal.load(Marshal.dump(lp))
+
+      expect(lp2.label).to eql(lp.label)
+      expect(lp2.features.values).to eql(lp.features.values)
+    end
+  end
+
+  context 'linear regression' do
+    context 'test' do
+      let(:lrm) { LinearRegressionWithSGD.train(rdd1) }
+
+      it 'test' do
+        expect(lrm.predict(values1[0])).to be <= 0
+        expect(lrm.predict(values1[1])).to be >  0
+        expect(lrm.predict(values1[2])).to be <= 0
+        expect(lrm.predict(values1[3])).to be >  0
+      end
+
+      it 'test via rdd' do
+        rdd = $sc.parallelize(values1, 1)
+        rdd = rdd.map(lambda{|value| model.predict(value)})
+        rdd = rdd.bind(model: lrm)
+
+        result = rdd.collect
+
+        expect(result[0]).to be <= 0
+        expect(result[1]).to be >  0
+        expect(result[2]).to be <= 0
+        expect(result[3]).to be >  0
+      end
+    end
+
+    # Y = 3 + 10*X1 + 10*X2
+    it 'linear regression' do
+      data = Spark.jb.call(RubyMLLibUtilAPI, 'generateLinearInput', 3.0, ['10.0', '10.0'], 100, 42, 0.1)
+      rdd = $sc.parallelize(data)
+
+      lrm = LinearRegressionWithSGD.train(rdd, iterations: 1000, intercept: true, step: 1.0)
+
+      expect(lrm.intercept).to be_between(2.5, 3.5)
+      expect(lrm.weights.size).to eq(2)
+      expect(lrm.weights[0]).to be_between(9.0, 11.0)
+      expect(lrm.weights[1]).to be_between(9.0, 11.0)
+    end
+  end
+
+  context 'lasso' do
+    it 'test' do
+      lrm = LassoWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+
+    it 'local random SGD with initial weights' do
+      data = Spark.jb.call(RubyMLLibUtilAPI, 'generateLinearInput', 2.0, ['-1.5', '0.01'], 1000, 42, 0.1)
+      data.map! do |lp|
+        LabeledPoint.new(lp.label, [1.0] + lp.features.values)
+      end
+
+      rdd = $sc.parallelize(data);
+
+      lrm = LassoWithSGD.train(rdd, step: 1.0, reg_param: 0.01, iterations: 40, initial_weights: [-1.0, -1.0, -1.0])
+
+      expect(lrm.weights[0]).to be_between(1.9, 2.1)
+      expect(lrm.weights[1]).to be_between(-1.60, -1.40)
+      expect(lrm.weights[2]).to be_between(-1.0e-3, 1.0e-3)
+    end
+  end
+
+  context 'ridge' do
+    it 'test' do
+      lrm = RidgeRegressionWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+end

data/spec/lib/mllib/vector_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib::Vector' do
+
+  context 'parsing' do
+    it 'dense vector' do
+      dv  = DenseVector.new([1.0, 2.0, 3.0, 4.0, 5.0])
+      dv2 = DenseVector.parse(dv.to_s)
+      dv3 = Vectors.parse(dv.to_s)
+
+      expect(dv.to_s).to eq("[1.0,2.0,3.0,4.0,5.0]")
+      expect(dv2.values).to eq(dv.values)
+      expect(dv3.values).to eq(dv.values)
+    end
+
+    it 'sparse vector' do
+      sv  = SparseVector.new(5, {1 => 3, 4 => 5})
+      sv2 = SparseVector.parse(sv.to_s)
+      sv3 = Vectors.parse(sv.to_s)
+
+      expect(sv.to_s).to eq("(5,[1,4],[3,5])")
+      expect(sv2.size).to eq(sv.size)
+      expect(sv2.indices).to eq(sv.indices)
+      expect(sv2.values).to eq(sv.values)
+      expect(sv3.size).to eq(sv.size)
+      expect(sv3.indices).to eq(sv.indices)
+      expect(sv3.values).to eq(sv.values)
+    end
+  end
+
+  it 'dot' do
+    sv = SparseVector.new(4, {1 => 1, 3 => 2})
+    dv = DenseVector.new([1.0, 2.0, 3.0, 4.0])
+    lst = DenseVector.new([1, 2, 3, 4])
+
+    expect(sv.dot(dv)).to eq(10.0)
+    expect(dv.dot(dv)).to eq(30.0)
+    expect(lst.dot(dv)).to eq(30.0)
+  end
+
+  it 'squared distance' do
+    sv = SparseVector.new(4, {1 => 1, 3 => 2})
+    dv = DenseVector.new([1.0, 2.0, 3.0, 4.0])
+    lst = DenseVector.new([4, 3, 2, 1])
+
+    expect(sv.squared_distance(dv)).to eq(15)
+    expect(sv.squared_distance(lst)).to eq(25)
+    expect(dv.squared_distance(lst)).to eq(20)
+    expect(dv.squared_distance(sv)).to eq(15)
+    expect(lst.squared_distance(sv)).to eq(25)
+    expect(lst.squared_distance(dv)).to eq(20)
+    expect(sv.squared_distance(sv)).to eq(0)
+    expect(dv.squared_distance(dv)).to eq(0)
+    expect(lst.squared_distance(lst)).to eq(0)
+  end
+
+  it 'sparse vector indexing' do
+    sv1 = SparseVector.new(4, {1 => 1, 3 => 2})
+    sv2 = SparseVector.new(4, [1, 3], [1, 2])
+
+    expect(sv1[0]).to eq(0)
+    expect(sv1[3]).to eq(2)
+    expect(sv1[1]).to eq(1)
+    expect(sv1[2]).to eq(0)
+    expect(sv1[-1]).to eq(2)
+    expect(sv1[-2]).to eq(0)
+    expect(sv1[-4]).to eq(0)
+
+    expect(sv2[0]).to eq(0)
+    expect(sv2[3]).to eq(2)
+    expect(sv2[1]).to eq(1)
+    expect(sv2[2]).to eq(0)
+    expect(sv2[-1]).to eq(2)
+    expect(sv2[-2]).to eq(0)
+    expect(sv2[-4]).to eq(0)
+  end
+end