ruby-spark 1.1.0.1-java

data/spec/inputs/numbers_1_100.txt

@@ -0,0 +1,100 @@
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100

data/spec/lib/collect_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+RSpec::describe Spark::RDD do
+
+  let(:mapping) { lambda{|x| [x, 1]} }
+  let(:numbers) { Generator.numbers }
+  
+  it ".collect_as_hash" do
+    rdd = $sc.parallelize(numbers)
+    rdd = rdd.map(mapping)
+
+    expect(rdd.collect_as_hash).to eql(Hash[numbers.map(&mapping)])
+  end
+
+  context ".take" do
+    let(:size)    { 1000 }
+    let(:numbers) { Generator.numbers(size) }
+    let(:rdd)     { $sc.parallelize(numbers) }
+
+    it "nothing" do
+      expect(rdd.take(0)).to eql([])
+    end
+
+    it "first" do
+      expect(rdd.first).to eql(numbers.first)
+    end
+
+    it "less than limit" do
+      _size = size / 2
+      expect(rdd.take(_size)).to eql(numbers.take(_size))
+    end
+
+    it "all" do
+      expect(rdd.take(size)).to eql(numbers)
+    end
+
+    it "more than limit" do
+      expect(rdd.take(size*2)).to eql(numbers)
+    end
+  end
+
+end

data/spec/lib/command_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+def to_s_method(x)
+  x.to_s
+end
+
+RSpec::describe Spark::CommandBuilder do
+  let(:numbers) { Generator.numbers }
+  let(:rdd)     { $sc.parallelize(numbers, 1) }
+
+  context '.serialize_function' do
+    let(:result)  { numbers.map(&:to_s) }
+
+    it 'string' do
+      expect(rdd.map('lambda{|x| x.to_s}').collect).to eql(result)
+    end
+
+    it 'symbol' do
+      expect(rdd.map(:to_s).collect).to eql(result)
+    end
+
+    it 'lambda' do
+      expect(rdd.map(lambda{|x| x.to_s}).collect).to eql(result)
+    end
+
+    it 'method' do
+      expect(rdd.map(method(:to_s_method)).collect).to eql(result)
+    end
+  end
+
+  context '.bind' do
+    it 'number' do
+      number = rand(0..10000000)
+      rdd2 = rdd.map(lambda{|x| x * number}).bind(number: number)
+
+      expect(rdd2.collect).to eq(numbers.map{|x| x * number})
+    end
+
+    it 'open struct' do
+      require 'ostruct'
+
+      struct = OpenStruct.new
+      struct.number = 3
+      struct.string = '3'
+      struct.array = [1, 2, 3]
+
+      func = lambda{|item|
+        item * struct.number + struct.string.to_i + struct.array[0]
+      }
+
+      rdd2 = rdd.add_library('ostruct')
+      rdd2 = rdd2.map(func)
+      rdd2 = rdd2.bind(struct: struct)
+
+      expect(rdd2.collect).to eq(numbers.map(&func))
+    end
+
+    it 'different naming' do
+      array = [1, 2, 3]
+
+      rdd2 = rdd.map(lambda{|_| my_array.size})
+      rdd2 = rdd2.bind(my_array: array)
+
+      expect(rdd2.sum).to eq(numbers.size * array.size)
+    end
+  end
+
+end

data/spec/lib/config_spec.rb

@@ -0,0 +1,64 @@
+require "spec_helper"
+
+RSpec::describe Spark::Config do
+
+  before(:context) do
+    Spark.stop
+  end
+
+  after(:context) do
+    spark_start
+  end
+
+  it "should be stopped" do
+    expect(Spark.started?).to be_falsy
+  end
+
+  context "new config" do
+
+    let(:configuration) do
+      {
+        "test.test1" => "test1",
+        "test.test2" => "test2",
+        "test.test3" => "test3"
+      }
+    end
+
+    before(:each) do
+      Spark.clear_config
+    end
+
+    it "throught methods" do
+      configuration.each do |key, value|
+        Spark.config.set(key, value)
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config.get(key)).to eql(value)
+      end
+    end
+
+    it "throught hash style" do
+      configuration.each do |key, value|
+        Spark.config[key] = value
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config[key]).to eql(value)
+      end
+    end
+
+    it "throught dsl" do
+      configuration.each do |key, value|
+        Spark.config {
+          set key, value
+        }
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config[key]).to eql(value)
+      end
+    end
+  end
+
+end

data/spec/lib/context_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+RSpec.describe Spark::Context do
+
+  it '.run_job' do
+    workers = 5
+    numbers = (0...100).to_a
+    func = lambda{|part| part.size}
+
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+    rdd = $sc.parallelize(numbers, workers, ser)
+
+    rdd_result = $sc.run_job(rdd, func)
+    result = numbers.each_slice(numbers.size/workers).map(&func)
+    expect(rdd_result).to eql(result)
+
+    parts = [0, 2]
+    func = lambda{|part| part.to_s}
+
+    rdd_result = $sc.run_job(rdd, func, parts)
+    result = []
+    sliced_numbers = numbers.each_slice(numbers.size/workers).to_a
+    parts.each do |part|
+      result << func.call(sliced_numbers[part])
+    end
+
+    expect(rdd_result).to eql(result)
+  end
+
+  it '.broadcast' do
+    workers = rand(1..5)
+
+    values1 = [1,2,3]
+    values2 = [4,5,6]
+
+    broadcast1 = $sc.broadcast(values1)
+    broadcast2 = $sc.broadcast(values2)
+
+    rdd = $sc.parallelize(0..5, workers)
+    rdd = rdd.bind(broadcast1: broadcast1, broadcast2: broadcast2)
+    rdd = rdd.map_partitions(lambda{|_| broadcast1.value + broadcast2.value })
+
+    expect(rdd.sum).to eql(
+      (values1 + values2).reduce(:+) * workers
+    )
+  end
+
+  context '.accumulator' do
+    it 'test' do
+      accum1 = $sc.accumulator(0,)
+      accum2 = $sc.accumulator(1, :*, 1)
+      accum3 = $sc.accumulator(0, lambda{|max, val| val > max ? val : max})
+
+      accum1 += 1
+
+      accum2.add(2)
+      accum2.add(2)
+      accum2.add(2)
+
+      accum3.add(9)
+      accum3.add(6)
+      accum3.add(7)
+
+      expect(accum1.value).to eql(1)
+      expect(accum2.value).to eql(8)
+      expect(accum3.value).to eql(9)
+
+      func = Proc.new do |_, index|
+        accum1.add(1)
+        accum2.add(2)
+        accum3.add(index * 10)
+      end
+
+      rdd = $sc.parallelize(0..4, 4)
+      rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3)
+      rdd = rdd.map_partitions_with_index(func)
+      rdd.collect
+
+      # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+      sleep(1)
+
+      expect(accum1.value).to eql(5)
+      expect(accum2.value).to eql(128)
+      expect(accum3.value).to eql(30)
+    end
+
+    context 'accum param' do
+      it 'symbol' do
+        accum1 = $sc.accumulator(1, :+, 0)
+        accum2 = $sc.accumulator(5, :-, 3)
+        accum3 = $sc.accumulator(1, :*, 1)
+        accum4 = $sc.accumulator(1.0, :/, 1.0)
+        accum5 = $sc.accumulator(2, :**, 2)
+
+        func = Proc.new do |_|
+          accum1.add(1)
+          accum2.add(1)
+          accum3.add(2)
+          accum4.add(2)
+          accum5.add(2)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3, accum4: accum4, accum5: accum5)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum1.value).to eq(3)
+        expect(accum2.value).to eq(1)
+        expect(accum3.value).to eq(4)
+        expect(accum4.value).to eq(4)
+        expect(accum5.value).to eq(65536)
+      end
+
+      it 'proc' do
+        accum1 = $sc.accumulator(1, lambda{|mem, val| mem + val}, 0)
+        accum2 = $sc.accumulator('a', lambda{|mem, val| mem + val}, '')
+        accum3 = $sc.accumulator([], lambda{|mem, val| mem << val}, [])
+
+        func = Proc.new do |_|
+          accum1.add(1)
+          accum2.add('a')
+          accum3.add(1)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum1.value).to eq(3)
+        expect(accum2.value).to eq('aaa')
+        expect(accum3.value).to eq([[1], [1]])
+      end
+
+      it 'string' do
+        expect { $sc.accumulator(1, '0') }.to raise_error(Spark::SerializeError)
+
+        accum = $sc.accumulator(1, 'lambda{|mem, val| mem + val}', 0)
+
+        func = Proc.new do |_|
+          accum.add(1)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum: accum)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum.value).to eq(3)
+      end
+    end
+  end
+
+end

data/spec/lib/ext_spec.rb

@@ -0,0 +1,72 @@
+require "spec_helper"
+
+RSpec.describe Array do
+
+  it ".deep_copy" do
+    data = ["a", "b", "c"]
+    new_data = data.dup
+
+    data[0] << "a"
+
+    expect(data).to eql(new_data)
+
+    new_data = data.deep_copy
+
+    data[1] << "b"
+
+    expect(data).to_not eql(new_data)
+  end
+
+end
+
+RSpec.describe Hash do
+
+  it ".stringify_keys!" do
+    data = {
+      a: "a",
+      b: "b",
+      c: "c"
+    }
+
+    data.stringify_keys!
+
+    expect(data).to eql({
+      "a" => "a",
+      "b" => "b",
+      "c" => "c"
+    })
+  end
+
+end
+
+RSpec.describe String do
+
+  it ".camelize" do
+    data = "aaa_bbb_ccc".camelize
+    expect(data).to eql("AaaBbbCcc")
+  end
+
+end
+
+RSpec.describe IO do
+
+  it 'serialize' do
+    file = Tempfile.new('serialize')
+    file.binmode
+
+    file.write_int(1)
+    file.write_long(2)
+    file.write_string("3")
+    file.write_data([4])
+
+    file.rewind
+
+    expect(file.read_int).to eq(1)
+    expect(file.read_long).to eq(2)
+    expect(file.read_string).to eq("3")
+    expect(file.read_data).to eq([4])
+
+    file.unlink
+  end
+
+end