ruby-spark 1.0.0

data/spec/inputs/numbers/8.txt

@@ -0,0 +1,50 @@
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400

data/spec/inputs/numbers/9.txt

@@ -0,0 +1,50 @@
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450

data/spec/inputs/numbers_0_100.txt

@@ -0,0 +1,101 @@
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100

data/spec/inputs/numbers_1_100.txt

@@ -0,0 +1,100 @@
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100

data/spec/lib/collect_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+RSpec::describe Spark::RDD do
+
+  let(:mapping) { lambda{|x| [x, 1]} }
+  let(:numbers) { Generator.numbers }
+  
+  it ".collect_as_hash" do
+    rdd = $sc.parallelize(numbers)
+    rdd = rdd.map(mapping)
+
+    expect(rdd.collect_as_hash).to eql(Hash[numbers.map(&mapping)])
+  end
+
+  context ".take" do
+    let(:size)    { 1000 }
+    let(:numbers) { Generator.numbers(size) }
+    let(:rdd)     { $sc.parallelize(numbers) }
+
+    it "nothing" do
+      expect(rdd.take(0)).to eql([])
+    end
+
+    it "first" do
+      expect(rdd.first).to eql(numbers.first)
+    end
+
+    it "less than limit" do
+      _size = size / 2
+      expect(rdd.take(_size)).to eql(numbers.take(_size))
+    end
+
+    it "all" do
+      expect(rdd.take(size)).to eql(numbers)
+    end
+
+    it "more than limit" do
+      expect(rdd.take(size*2)).to eql(numbers)
+    end
+  end
+
+end

data/spec/lib/command_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+def to_s_method(x)
+  x.to_s
+end
+
+RSpec::describe Spark::CommandBuilder do
+  let(:numbers) { Generator.numbers }
+  let(:rdd)     { $sc.parallelize(numbers, 1) }
+
+  context '.serialize_function' do
+    let(:result)  { numbers.map(&:to_s) }
+
+    it 'string' do
+      expect(rdd.map('lambda{|x| x.to_s}').collect).to eql(result)
+    end
+
+    it 'symbol' do
+      expect(rdd.map(:to_s).collect).to eql(result)
+    end
+
+    it 'lambda' do
+      expect(rdd.map(lambda{|x| x.to_s}).collect).to eql(result)
+    end
+
+    it 'method' do
+      expect(rdd.map(method(:to_s_method)).collect).to eql(result)
+    end
+  end
+
+  context '.bind' do
+    it 'number' do
+      number = rand(0..10000000)
+      rdd2 = rdd.map(lambda{|x| x * number}).bind(number: number)
+
+      expect(rdd2.collect).to eq(numbers.map{|x| x * number})
+    end
+
+    it 'open struct' do
+      require 'ostruct'
+
+      struct = OpenStruct.new
+      struct.number = 3
+      struct.string = '3'
+      struct.array = [1, 2, 3]
+
+      func = lambda{|item|
+        item * struct.number + struct.string.to_i + struct.array[0]
+      }
+
+      rdd2 = rdd.add_library('ostruct')
+      rdd2 = rdd2.map(func)
+      rdd2 = rdd2.bind(struct: struct)
+
+      expect(rdd2.collect).to eq(numbers.map(&func))
+    end
+
+    it 'different naming' do
+      array = [1, 2, 3]
+
+      rdd2 = rdd.map(lambda{|_| my_array.size})
+      rdd2 = rdd2.bind(my_array: array)
+
+      expect(rdd2.sum).to eq(numbers.size * array.size)
+    end
+  end
+
+end

data/spec/lib/config_spec.rb

@@ -0,0 +1,64 @@
+require "spec_helper"
+
+RSpec::describe Spark::Config do
+
+  before(:context) do
+    Spark.stop
+  end
+
+  after(:context) do
+    spark_start
+  end
+
+  it "should be stopped" do
+    expect(Spark.started?).to be_falsy
+  end
+
+  context "new config" do
+
+    let(:configuration) do
+      {
+        "test.test1" => "test1",
+        "test.test2" => "test2",
+        "test.test3" => "test3"
+      }
+    end
+
+    before(:each) do
+      Spark.clear_config
+    end
+
+    it "throught methods" do
+      configuration.each do |key, value|
+        Spark.config.set(key, value)
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config.get(key)).to eql(value)
+      end
+    end
+
+    it "throught hash style" do
+      configuration.each do |key, value|
+        Spark.config[key] = value
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config[key]).to eql(value)
+      end
+    end
+
+    it "throught dsl" do
+      configuration.each do |key, value|
+        Spark.config {
+          set key, value
+        }
+      end
+
+      configuration.each do |key, value|
+        expect(Spark.config[key]).to eql(value)
+      end
+    end
+  end
+
+end