ruby-spark 1.0.0

data/spec/lib/mllib/classification_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib classification' do
+
+  let(:data1) do
+    [
+      LabeledPoint.new(0.0, [1, 0, 0]),
+      LabeledPoint.new(1.0, [0, 1, 1]),
+      LabeledPoint.new(0.0, [2, 0, 0]),
+      LabeledPoint.new(1.0, [0, 2, 1])
+    ]
+  end
+
+  let(:values1) do
+    data1.map do |lp|
+      lp.features.values
+    end
+  end
+
+  let(:rdd1) { $sc.parallelize(data1) }
+
+  context 'logistic regression' do
+    it 'test' do
+      lrm = LogisticRegressionWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+
+  context 'svm' do
+    it 'test' do
+      lrm = SVMWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+
+  context 'naive bayes' do
+    it 'test' do
+      lrm = NaiveBayes.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+end

data/spec/lib/mllib/clustering_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib clustering' do
+  context 'kmeans' do
+    it 'test' do
+      data = [
+        DenseVector.new([0, 1.1]),
+        DenseVector.new([0, 1.2]),
+        DenseVector.new([1.1, 0]),
+        DenseVector.new([1.2, 0])
+      ]
+      model = KMeans.train($sc.parallelize(data), 2, initialization_mode: 'k-means||')
+
+      expect(model.predict(data[0])).to eq(model.predict(data[1]))
+      expect(model.predict(data[2])).to eq(model.predict(data[3]))
+    end
+
+    it 'deterministic' do
+      data = Array.new(10) do |i|
+        i *= 10
+        DenseVector.new([i, i])
+      end
+
+      clusters1 = KMeans.train($sc.parallelize(data), 3, initialization_mode: 'k-means||', seed: 42)
+      clusters2 = KMeans.train($sc.parallelize(data), 3, initialization_mode: 'k-means||', seed: 42)
+
+      centers1 = clusters1.centers.to_a
+      centers2 = clusters2.centers.to_a
+
+      centers1.zip(centers2).each do |c1, c2|
+        expect(c1).to eq(c2)
+      end
+    end
+  end
+end

data/spec/lib/mllib/matrix_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib::Matrix' do
+  context 'dense' do
+    it 'construct' do
+      values = [[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]]
+      matrix = DenseMatrix.new(3, 3, [[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]])
+
+      expect(matrix.shape).to eq([3, 3])
+      expect(matrix.values).to eq([[1.0, 0.0, 4.0], [0.0, 3.0, 5.0], [2.0, 0.0, 6.0]])
+    end
+  end
+
+  context 'sparse' do
+    it 'construct' do
+      values = [1.0, 2.0, 4.0, 5.0]
+      column_pointers = [0, 2, 2, 4, 4]
+      row_indices = [1, 2, 1, 2]
+
+      matrix = SparseMatrix.new(3, 4, column_pointers, row_indices, values)
+
+      expect(matrix.shape).to eq([3, 4])
+      expect(matrix.to_a).to eq(
+        [
+          [0.0, 0.0, 0.0, 0.0],
+          [1.0, 0.0, 4.0, 0.0],
+          [2.0, 0.0, 5.0, 0.0]
+        ]
+      )
+    end
+  end
+end

data/spec/lib/mllib/regression_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+
+# Mllib functions are tested on Spark
+# This just test if ruby call proper methods
+
+RSpec.describe 'Spark::Mllib regression' do
+
+  let(:data1) do
+    [
+      LabeledPoint.new(-1.0, [0, -1]),
+      LabeledPoint.new(1.0, [0, 1]),
+      LabeledPoint.new(-1.0, [0, -2]),
+      LabeledPoint.new(1.0, [0, 2])
+    ]
+  end
+
+  let(:values1) do
+    data1.map do |lp|
+      lp.features.values
+    end
+  end
+
+  let(:rdd1) { $sc.parallelize(data1) }
+
+  context 'labeled point' do
+    let(:lp) { LabeledPoint.new(1, [1,2,3]) }
+
+    it 'from array' do
+      expect(lp.label).to eql(1.0)
+      expect(lp.features).to be_a(DenseVector)
+    end
+
+    it 'serialize' do
+      lp2 = Marshal.load(Marshal.dump(lp))
+
+      expect(lp2.label).to eql(lp.label)
+      expect(lp2.features.values).to eql(lp.features.values)
+    end
+  end
+
+  context 'linear regression' do
+    context 'test' do
+      let(:lrm) { LinearRegressionWithSGD.train(rdd1) }
+
+      it 'test' do
+        expect(lrm.predict(values1[0])).to be <= 0
+        expect(lrm.predict(values1[1])).to be >  0
+        expect(lrm.predict(values1[2])).to be <= 0
+        expect(lrm.predict(values1[3])).to be >  0
+      end
+
+      it 'test via rdd' do
+        rdd = $sc.parallelize(values1, 1)
+        rdd = rdd.map(lambda{|value| model.predict(value)})
+        rdd = rdd.bind(model: lrm)
+
+        result = rdd.collect
+
+        expect(result[0]).to be <= 0
+        expect(result[1]).to be >  0
+        expect(result[2]).to be <= 0
+        expect(result[3]).to be >  0
+      end
+    end
+
+    # Y = 3 + 10*X1 + 10*X2
+    it 'linear regression' do
+      data = Spark.jb.call(RubyMLLibUtilAPI, 'generateLinearInput', 3.0, ['10.0', '10.0'], 100, 42, 0.1)
+      rdd = $sc.parallelize(data)
+
+      lrm = LinearRegressionWithSGD.train(rdd, iterations: 1000, intercept: true, step: 1.0)
+
+      expect(lrm.intercept).to be_between(2.5, 3.5)
+      expect(lrm.weights.size).to eq(2)
+      expect(lrm.weights[0]).to be_between(9.0, 11.0)
+      expect(lrm.weights[1]).to be_between(9.0, 11.0)
+    end
+  end
+
+  context 'lasso' do
+    it 'test' do
+      lrm = LassoWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+
+    it 'local random SGD with initial weights' do
+      data = Spark.jb.call(RubyMLLibUtilAPI, 'generateLinearInput', 2.0, ['-1.5', '0.01'], 1000, 42, 0.1)
+      data.map! do |lp|
+        LabeledPoint.new(lp.label, [1.0] + lp.features.values)
+      end
+
+      rdd = $sc.parallelize(data);
+
+      lrm = LassoWithSGD.train(rdd, step: 1.0, reg_param: 0.01, iterations: 40, initial_weights: [-1.0, -1.0, -1.0])
+
+      expect(lrm.weights[0]).to be_between(1.9, 2.1)
+      expect(lrm.weights[1]).to be_between(-1.60, -1.40)
+      expect(lrm.weights[2]).to be_between(-1.0e-3, 1.0e-3)
+    end
+  end
+
+  context 'ridge' do
+    it 'test' do
+      lrm = RidgeRegressionWithSGD.train(rdd1)
+
+      expect(lrm.predict(values1[0])).to be <= 0
+      expect(lrm.predict(values1[1])).to be >  0
+      expect(lrm.predict(values1[2])).to be <= 0
+      expect(lrm.predict(values1[3])).to be >  0
+    end
+  end
+end

data/spec/lib/mllib/vector_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+RSpec.describe 'Spark::Mllib::Vector' do
+
+  context 'parsing' do
+    it 'dense vector' do
+      dv  = DenseVector.new([1.0, 2.0, 3.0, 4.0, 5.0])
+      dv2 = DenseVector.parse(dv.to_s)
+      dv3 = Vectors.parse(dv.to_s)
+
+      expect(dv.to_s).to eq("[1.0,2.0,3.0,4.0,5.0]")
+      expect(dv2.values).to eq(dv.values)
+      expect(dv3.values).to eq(dv.values)
+    end
+
+    it 'sparse vector' do
+      sv  = SparseVector.new(5, {1 => 3, 4 => 5})
+      sv2 = SparseVector.parse(sv.to_s)
+      sv3 = Vectors.parse(sv.to_s)
+
+      expect(sv.to_s).to eq("(5,[1,4],[3,5])")
+      expect(sv2.size).to eq(sv.size)
+      expect(sv2.indices).to eq(sv.indices)
+      expect(sv2.values).to eq(sv.values)
+      expect(sv3.size).to eq(sv.size)
+      expect(sv3.indices).to eq(sv.indices)
+      expect(sv3.values).to eq(sv.values)
+    end
+  end
+
+  it 'dot' do
+    sv = SparseVector.new(4, {1 => 1, 3 => 2})
+    dv = DenseVector.new([1.0, 2.0, 3.0, 4.0])
+    lst = DenseVector.new([1, 2, 3, 4])
+
+    expect(sv.dot(dv)).to eq(10.0)
+    expect(dv.dot(dv)).to eq(30.0)
+    expect(lst.dot(dv)).to eq(30.0)
+  end
+
+  it 'squared distance' do
+    sv = SparseVector.new(4, {1 => 1, 3 => 2})
+    dv = DenseVector.new([1.0, 2.0, 3.0, 4.0])
+    lst = DenseVector.new([4, 3, 2, 1])
+
+    expect(sv.squared_distance(dv)).to eq(15)
+    expect(sv.squared_distance(lst)).to eq(25)
+    expect(dv.squared_distance(lst)).to eq(20)
+    expect(dv.squared_distance(sv)).to eq(15)
+    expect(lst.squared_distance(sv)).to eq(25)
+    expect(lst.squared_distance(dv)).to eq(20)
+    expect(sv.squared_distance(sv)).to eq(0)
+    expect(dv.squared_distance(dv)).to eq(0)
+    expect(lst.squared_distance(lst)).to eq(0)
+  end
+
+  it 'sparse vector indexing' do
+    sv1 = SparseVector.new(4, {1 => 1, 3 => 2})
+    sv2 = SparseVector.new(4, [1, 3], [1, 2])
+
+    expect(sv1[0]).to eq(0)
+    expect(sv1[3]).to eq(2)
+    expect(sv1[1]).to eq(1)
+    expect(sv1[2]).to eq(0)
+    expect(sv1[-1]).to eq(2)
+    expect(sv1[-2]).to eq(0)
+    expect(sv1[-4]).to eq(0)
+
+    expect(sv2[0]).to eq(0)
+    expect(sv2[3]).to eq(2)
+    expect(sv2[1]).to eq(1)
+    expect(sv2[2]).to eq(0)
+    expect(sv2[-1]).to eq(2)
+    expect(sv2[-2]).to eq(0)
+    expect(sv2[-4]).to eq(0)
+  end
+end

data/spec/lib/reduce_by_key_spec.rb

@@ -0,0 +1,118 @@
+require "spec_helper"
+
+def flat_map(line)
+  line.split
+end
+
+def map(item)
+  [item, 1]
+end
+
+def reduce(x,y)
+  x+y
+end
+
+RSpec::shared_examples "a words counting" do |workers|
+  context "with #{workers || 'default'} worker" do
+    let(:result) do
+      keyyed = lines.flat_map{|x| x.split}.map{|x| [x,1]}
+      result = keyyed.reduce({}){|memo, item|
+        key   = item[0]
+        value = item[1]
+
+        memo[key] ||= 0
+        memo[key] += value
+        memo
+      }
+      result
+    end
+
+    it "when lambda" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(lambda{|line| line.split})
+      rdd2 = rdd2.map(lambda{|word| [word, 1]})
+      rdd2 = rdd2.reduce_by_key(lambda{|x,y| x+y})
+
+      expect(rdd2.collect_as_hash).to eql(result)
+    end
+
+    it "when method" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(method(:flat_map))
+      rdd2 = rdd2.map(method(:map))
+      rdd2 = rdd2.reduce_by_key(method(:reduce))
+
+      expect(rdd2.collect_as_hash).to eql(result)
+    end
+
+    it "keys, values" do
+      rdd2 = rdd(workers)
+      rdd2 = rdd2.flat_map(method(:flat_map))
+      rdd2 = rdd2.map(method(:map))
+      rdd2 = rdd2.reduce_by_key(method(:reduce))
+
+      expect(rdd2.keys.collect.sort).to eql(result.keys.sort)
+      expect { rdd2.values.collect.reduce(:+) }.to_not raise_error
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  context ".reduce_by_key" do
+    context "throught parallelize" do
+      let(:lines) { Generator.lines }
+
+      def rdd(workers)
+        $sc.parallelize(lines, workers)
+      end
+
+      it_behaves_like "a words counting", nil
+      it_behaves_like "a words counting", 1
+      it_behaves_like "a words counting", rand(2..10)
+    end
+
+    context "throught text_file" do
+      let(:file)  { File.join("spec", "inputs", "lorem_300.txt") }
+      let(:lines) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a words counting", nil
+      it_behaves_like "a words counting", 1
+      it_behaves_like "a words counting", rand(2..10)
+    end
+  end
+
+  context ".fold_by_key" do
+    let(:numbers)    { Generator.numbers }
+    let(:zero_value) { 0 }
+    let(:rdd)        { $sc.parallelize(numbers) }
+    let(:map)        { lambda{|x| [x, 1]} }
+    let(:add)        { lambda{|x,y| x+y} }
+
+    let(:result) do
+      _result = {}
+      numbers.map(&map).each do |key, value|
+        _result[key] ||= zero_value
+        _result[key] = add.call(_result[key], value)
+      end
+      _result
+    end
+
+    def fold_by_key(num_partitions=nil)
+      rdd.map(map).fold_by_key(zero_value, add, num_partitions).collect_as_hash
+    end
+
+    it "default num_partitions" do
+      expect(fold_by_key).to eq(result)
+    end
+
+    it "default num_partitions" do
+      expect(
+        fold_by_key rand(1..10)
+      ).to eq(result)
+    end
+  end
+end

data/spec/lib/reduce_spec.rb

@@ -0,0 +1,131 @@
+require "spec_helper"
+
+def longest_words(memo, word)
+  memo.length > word.length ? memo : word
+end
+
+RSpec::shared_examples "a reducing" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it ".reduce" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+      rdd2 = rdd2.reduce(func1)
+      result = numbers.map(&:to_i).reduce(&func1)
+
+      expect(rdd2).to eql(result)
+
+      rdd3 = rdd_numbers(workers)
+      rdd3 = rdd3.map(to_i)
+      rdd3 = rdd3.reduce(func2)
+      result = numbers.map(&:to_i).reduce(&func2)
+
+      expect(rdd3).to eql(result)
+
+      rdd4 = rdd_lines(workers)
+      rdd4 = rdd4.flat_map(split)
+      rdd4 = rdd4.reduce(method(:longest_words))
+
+      result = lines.flat_map(&split).reduce(&lambda(&method(:longest_words)))
+
+      expect(rdd4).to eql(result)
+    end
+
+    it ".fold" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+      rdd_result = rdd2.fold(1, func1)
+
+      # all workers add 1 + last reducing phase
+      result = numbers.map(&:to_i).reduce(&func1) + rdd2.partitions_size + 1
+
+      expect(rdd_result).to eql(result)
+    end
+
+    it ".aggregate" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      # Sum of items + their count
+      seq = lambda{|x,y| [x[0] + y, x[1] + 1]}
+      com = lambda{|x,y| [x[0] + y[0], x[1] + y[1]]}
+      rdd_result = rdd2.aggregate([0,0], seq, com)
+
+      result = [numbers.reduce(:+), numbers.size]
+
+      expect(rdd_result).to eql(result)
+    end
+
+    it ".max" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.max).to eql(numbers.map(&:to_i).max)
+    end
+
+    it ".min" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.min).to eql(numbers.map(&:to_i).min)
+    end
+
+    it ".sum" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.sum).to eql(numbers.map(&:to_i).reduce(:+))
+    end
+
+    it ".count" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.map(to_i)
+
+      expect(rdd2.count).to eql(numbers.size)
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|sum, x| sum+x} }
+  let(:func2) { lambda{|product, x| product*x} }
+
+  let(:to_i)  { lambda{|item| item.to_i} }
+  let(:split) { lambda{|item| item.split} }
+
+  context "throught parallelize" do
+    let(:numbers) { Generator.numbers }
+    let(:lines)   { Generator.lines }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    def rdd_lines(workers)
+      $sc.parallelize(lines, workers)
+    end
+
+    it_behaves_like "a reducing", nil
+    it_behaves_like "a reducing", 1
+    it_behaves_like "a reducing", rand(2..10)
+  end
+
+  context "throught text_file" do
+    let(:file)       { File.join("spec", "inputs", "numbers_0_100.txt") }
+    let(:file_lines) { File.join("spec", "inputs", "lorem_300.txt") }
+
+    let(:numbers) { File.readlines(file).map(&:strip).map(&:to_i) }
+    let(:lines)   { File.readlines(file_lines).map(&:strip) }
+
+    def rdd_numbers(workers)
+      $sc.text_file(file, workers)
+    end
+
+    def rdd_lines(workers)
+      $sc.text_file(file_lines, workers)
+    end
+
+    it_behaves_like "a reducing", nil
+    it_behaves_like "a reducing", 1
+    it_behaves_like "a reducing", rand(2..10)
+  end
+end