ruby-spark 1.1.0.1 → 1.2.0

data/spec/lib/flat_map_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a flat mapping" do |workers|
+RSpec.shared_examples 'a flat mapping' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers).map(func1)
     result = numbers.flat_map(&func1)
@@ -24,7 +24,7 @@ RSpec::shared_examples "a flat mapping" do |workers|
   end
 end
 
-RSpec::shared_examples "a flat mapping values" do |workers|
+RSpec.shared_examples 'a flat mapping values' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers).flat_map_values(func1)
     result = []
@@ -50,25 +50,26 @@ RSpec::shared_examples "a flat mapping values" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:func1) { lambda{|x| x*2} }
   let(:func2) { lambda{|x| [x*3, 1, 1]} }
   let(:func3) { lambda{|x| [x*4, 2, 2]} }
 
-  context "throught parallelize" do
-    context ".flat_map" do
+  context 'throught parallelize' do
+    context '.flat_map' do
       let(:numbers) { Generator.numbers_with_zero }
 
       def rdd(workers)
         $sc.parallelize(numbers, workers)
       end
 
-      it_behaves_like "a flat mapping", nil
-      it_behaves_like "a flat mapping", 1
-      it_behaves_like "a flat mapping", rand(2..10)
+      it_behaves_like 'a flat mapping', 1
+      it_behaves_like 'a flat mapping', 2
+      # it_behaves_like 'a flat mapping', nil
+      # it_behaves_like 'a flat mapping', rand(2..10)
     end
 
-    context ".flat_map_values" do
+    context '.flat_map_values' do
       let(:func1) { lambda{|x| x*2} }
       let(:func2) { lambda{|x| [x.first]} }
       let(:hash_with_values) { Generator.hash_with_values }
@@ -77,24 +78,26 @@ RSpec::describe "Spark::RDD" do
         $sc.parallelize(hash_with_values, workers)
       end
 
-      it_behaves_like "a flat mapping values", nil
-      it_behaves_like "a flat mapping values", 1
-      it_behaves_like "a flat mapping values", rand(2..10)
+      it_behaves_like 'a flat mapping values', 1
+      it_behaves_like 'a flat mapping values', 2
+      # it_behaves_like 'a flat mapping values', nil
+      # it_behaves_like 'a flat mapping values', rand(2..10)
     end
   end
 
-  context "throught text_file" do
-    context ".flat_map" do
-      let(:file)    { File.join("spec", "inputs", "numbers_0_100.txt") }
+  context 'throught text_file' do
+    context '.flat_map' do
+      let(:file)    { File.join('spec', 'inputs', 'numbers_0_100.txt') }
       let(:numbers) { File.readlines(file).map(&:strip) }
 
       def rdd(workers)
         $sc.text_file(file, workers)
       end
 
-      it_behaves_like "a flat mapping", nil
-      it_behaves_like "a flat mapping", 1
-      it_behaves_like "a flat mapping", rand(2..10)
+      it_behaves_like 'a flat mapping', 1
+      it_behaves_like 'a flat mapping', 2
+      # it_behaves_like 'a flat mapping', nil
+      # it_behaves_like 'a flat mapping', rand(2..10)
     end
   end
 end

data/spec/lib/group_spec.rb

@@ -1,26 +1,26 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a groupping by key" do |workers|
+RSpec.shared_examples 'a groupping by key' do |workers|
   it "with #{workers || 'default'} worker" do
     expect(rdd_result(workers)).to eql(result)
   end
 end
 
-RSpec::shared_examples "a cogroupping by key" do |workers|
+RSpec.shared_examples 'a cogroupping by key' do |workers|
   context "with #{workers || 'default'} worker" do
-    it ".group_with" do
+    it '.group_with' do
       rdd = rdd_1(workers).group_with(rdd_2(workers))
       expect(rdd.collect_as_hash).to eql(result_12)
     end
 
-    it ".cogroup" do
+    it '.cogroup' do
       rdd = rdd_1(workers).cogroup(rdd_2(workers), rdd_3(workers))
       expect(rdd.collect_as_hash).to eql(result_123)
     end
   end
 end
 
-RSpec::shared_examples "a groupping by" do |workers|
+RSpec.shared_examples 'a groupping by' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd = rdd_numbers(workers)
     rdd = rdd.group_by(key_function1)
@@ -34,7 +34,7 @@ RSpec::shared_examples "a groupping by" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
 
   def make_result(*hashes)
     _result = {}
@@ -47,7 +47,7 @@ RSpec::describe "Spark::RDD" do
     _result
   end
 
-  context ".group_by_key" do
+  context '.group_by_key' do
     let(:hash) { Generator.hash }
     let(:result) { make_result(hash) }
 
@@ -56,12 +56,13 @@ RSpec::describe "Spark::RDD" do
       rdd.group_by_key.collect_as_hash
     end
 
-    it_behaves_like "a groupping by key", nil
-    it_behaves_like "a groupping by key", 1
-    it_behaves_like "a groupping by key", rand(2..10)
+    it_behaves_like 'a groupping by key', 1
+    it_behaves_like 'a groupping by key', 2
+    # it_behaves_like 'a groupping by key', nil
+    # it_behaves_like 'a groupping by key', rand(2..10)
   end
 
-  context "cogroup" do
+  context 'cogroup' do
     let(:hash1) { Generator.hash }
     let(:hash2) { Generator.hash }
     let(:hash3) { Generator.hash }
@@ -81,12 +82,13 @@ RSpec::describe "Spark::RDD" do
       $sc.parallelize(hash3)
     end
 
-    it_behaves_like "a cogroupping by key", nil
-    it_behaves_like "a cogroupping by key", 1
-    it_behaves_like "a cogroupping by key", rand(2..10)
+    it_behaves_like 'a cogroupping by key', 1
+    it_behaves_like 'a cogroupping by key', 2
+    # it_behaves_like 'a cogroupping by key', nil
+    # it_behaves_like 'a cogroupping by key', rand(2..10)
   end
 
-  context "group_by" do
+  context 'group_by' do
     let(:key_function1) { lambda{|x| x%2} }
     let(:key_function2) { lambda{|x| x.size} }
 
@@ -101,9 +103,10 @@ RSpec::describe "Spark::RDD" do
       $sc.parallelize(words)
     end
 
-    it_behaves_like "a groupping by", nil
-    it_behaves_like "a groupping by", 1
-    it_behaves_like "a groupping by", rand(2..10)
+    it_behaves_like 'a groupping by', 1
+    it_behaves_like 'a groupping by', 2
+    # it_behaves_like 'a groupping by', nil
+    # it_behaves_like 'a groupping by', rand(2..10)
   end
 
 end

data/spec/lib/helper_spec.rb

@@ -1,19 +1,67 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::configure do |c|
+RSpec.configure do |c|
   c.include Spark::Helper::Parser
+  c.include Spark::Helper::Statistic
 end
 
-RSpec::describe Spark::Helper do
-  
-  it "memory size" do
-    expect(to_memory_size("512mb")).to eql(524288.0)
-    expect(to_memory_size("1586 mb")).to eql(1624064.0)
-    expect(to_memory_size("3 MB")).to eql(3072.0)
-    expect(to_memory_size("9gb")).to eql(9437184.0)
-    expect(to_memory_size("9gb", "mb")).to eql(9216.0)
-    expect(to_memory_size("9mb", "gb")).to eql(0.01)
-    expect(to_memory_size("6652548796kb", "mb")).to eql(6496629.68)
+RSpec.describe Spark::Helper do
+
+  it 'memory size' do
+    expect(to_memory_size('512mb')).to eql(524288.0)
+    expect(to_memory_size('1586 mb')).to eql(1624064.0)
+    expect(to_memory_size('3 MB')).to eql(3072.0)
+    expect(to_memory_size('9gb')).to eql(9437184.0)
+    expect(to_memory_size('9gb', 'mb')).to eql(9216.0)
+    expect(to_memory_size('9mb', 'gb')).to eql(0.01)
+    expect(to_memory_size('6652548796kb', 'mb')).to eql(6496629.68)
+  end
+
+  context 'statistic' do
+    it 'compute_fraction' do
+      expect(compute_fraction(1, 1000, true)).to be_within(0.001).of(0.013)
+      expect(compute_fraction(2, 1000, true)).to be_within(0.001).of(0.018)
+      expect(compute_fraction(3, 1000, true)).to be_within(0.001).of(0.023)
+      expect(compute_fraction(4, 1000, true)).to be_within(0.001).of(0.028)
+      expect(compute_fraction(5, 1000, true)).to be_within(0.001).of(0.031)
+
+      expect(compute_fraction(1, 1000, false)).to be_within(0.001).of(0.0249)
+      expect(compute_fraction(2, 1000, false)).to be_within(0.001).of(0.0268)
+      expect(compute_fraction(3, 1000, false)).to be_within(0.001).of(0.0287)
+      expect(compute_fraction(4, 1000, false)).to be_within(0.001).of(0.0305)
+      expect(compute_fraction(5, 1000, false)).to be_within(0.001).of(0.0322)
+    end
+
+    it 'bisect_right' do
+      data = [10, 20, 30, 40, 50, 60, 70, 80, 90]
+
+      expect(bisect_right(data, 0)).to eq(0)
+      expect(bisect_right(data, 1)).to eq(0)
+      expect(bisect_right(data, 1, 2)).to eq(2)
+      expect(bisect_right(data, 1, 3)).to eq(3)
+      expect(bisect_right(data, 1, 4)).to eq(4)
+      expect(bisect_right(data, 9)).to eq(0)
+      expect(bisect_right(data, 10)).to eq(1)
+      expect(bisect_right(data, 40)).to eq(4)
+      expect(bisect_right(data, 42)).to eq(4)
+      expect(bisect_right(data, 72)).to eq(7)
+      expect(bisect_right(data, 80, 4)).to eq(8)
+      expect(bisect_right(data, 80, 5)).to eq(8)
+      expect(bisect_right(data, 80, 8)).to eq(8)
+      expect(bisect_right(data, 80, 9)).to eq(9)
+      expect(bisect_right(data, 200)).to eq(9)
+    end
+
+    it 'determine_bounds' do
+      data = [10, 20, 30, 40, 50, 60, 70, 80, 90]
+
+      expect(determine_bounds(data, 0)).to eq([])
+      expect(determine_bounds(data, 1)).to eq([])
+      expect(determine_bounds(data, 2)).to eq([50])
+      expect(determine_bounds(data, 3)).to eq([40, 70])
+      expect(determine_bounds(data, 4)).to eq([30, 50, 70])
+      expect(determine_bounds(data, 20)).to eq(data)
+    end
   end
 
 end

data/spec/lib/key_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a keying by" do |workers|
+RSpec.shared_examples 'a keying by' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd = rdd_numbers(workers)
     rdd = rdd.key_by(key_function1)
@@ -16,11 +16,11 @@ RSpec::shared_examples "a keying by" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
 
-  context "key_by" do
+  context 'key_by' do
     let(:key_function1) { lambda{|x| x.even?} }
-    let(:key_function2) { lambda{|x| x.include?("a")} }
+    let(:key_function2) { lambda{|x| x.include?('a')} }
 
     let(:numbers) { Generator.numbers }
     let(:words)   { Generator.words }
@@ -33,9 +33,10 @@ RSpec::describe "Spark::RDD" do
       $sc.parallelize(words)
     end
 
-    it_behaves_like "a keying by", nil
-    it_behaves_like "a keying by", 1
-    it_behaves_like "a keying by", rand(2..10)
+    it_behaves_like 'a keying by', 1
+    it_behaves_like 'a keying by', 2
+    # it_behaves_like 'a keying by', nil
+    # it_behaves_like 'a keying by', rand(2..10)
   end
 
 end

data/spec/lib/manipulation_spec.rb

@@ -1,10 +1,10 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:numbers) { 1..100 }
   let(:rand_numbers) { Generator.numbers }
 
-  it ".glom" do
+  it '.glom' do
     rdd = $sc.parallelize(numbers, 1).glom
     expect(rdd.collect).to eql([numbers.to_a])
 
@@ -14,7 +14,7 @@ RSpec::describe "Spark::RDD" do
     expect(rdd.collect).to eql(numbers.each_slice(20).to_a)
   end
 
-  it ".coalesce" do
+  it '.coalesce' do
     rdd = $sc.parallelize(numbers, 5)
 
     rdd2 = rdd.glom
@@ -24,7 +24,7 @@ RSpec::describe "Spark::RDD" do
     expect(rdd3.collect.size).to eql(4)
   end
 
-  it ".distinct" do
+  it '.distinct' do
     rdd = $sc.parallelize(rand_numbers, 5)
     rdd = rdd.distinct
     expect(rdd.collect.sort).to eql(rand_numbers.uniq.sort)
@@ -35,22 +35,22 @@ RSpec::describe "Spark::RDD" do
     expect(rdd.collect).to eql([1])
   end
 
-  context ".union" do
-    it "classic method" do
+  context '.union' do
+    it 'classic method' do
       rdd = $sc.parallelize(numbers, 5)
       rdd = rdd.union(rdd).collect
 
       expect(rdd.collect.sort).to eql((numbers.to_a+numbers.to_a).sort)
     end
 
-    it "with a different serializer" do
+    it 'with a different serializer' do
       rdd1 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__marshal__) })
       rdd2 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__oj__) })
 
       expect { rdd1.union(rdd2).collect }.to_not raise_error
     end
 
-    it "as operator" do
+    it 'as operator' do
       rdd1 = $sc.parallelize(numbers)
       rdd2 = $sc.parallelize(rand_numbers)
 
@@ -58,7 +58,7 @@ RSpec::describe "Spark::RDD" do
     end
   end
 
-  it ".compact" do
+  it '.compact' do
     data = [nil, nil , 0, 0, 1, 2, nil, 6]
     result = data.compact
     ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
@@ -73,7 +73,7 @@ RSpec::describe "Spark::RDD" do
     expect(rdd.collect).to eql(result)
   end
 
-  it ".intersection" do
+  it '.intersection' do
     data1 = [0,1,2,3,4,5,6,7,8,9,10]
     data2 = [5,6,7,8,9,10,11,12,13,14,15]
 
@@ -83,19 +83,19 @@ RSpec::describe "Spark::RDD" do
     expect(rdd1.intersection(rdd2).collect.sort).to eql(data1 & data2)
   end
 
-  it ".shuffle" do
+  it '.shuffle' do
     data = Generator.numbers
     rdd = $sc.parallelize(data)
 
     expect(rdd.shuffle.collect).to_not eql(data)
   end
 
-  context ".cartesian" do
+  context '.cartesian' do
     let(:data1) { Generator.numbers(100) }
     let(:data2) { Generator.numbers(100) }
     let(:result) { data1.product(data2).map(&:to_s).sort }
 
-    it "unbatched" do
+    it 'unbatched' do
       ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
 
       rdd1 = $sc.parallelize(data1, 2, ser)
@@ -106,7 +106,7 @@ RSpec::describe "Spark::RDD" do
       expect(rdd.collect.sort).to eql(result)
     end
 
-    it "batched" do
+    it 'batched' do
       ser1 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
       ser2 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
 

data/spec/lib/map_partitions_spec.rb

@@ -67,9 +67,10 @@ RSpec::describe 'Spark::RDD.map_partitions(_with_index)' do
       $sc.parallelize(numbers, workers)
     end
 
-    it_behaves_like 'a map partitions', nil
     it_behaves_like 'a map partitions', 1
-    it_behaves_like 'a map partitions', rand(2..10)
+    it_behaves_like 'a map partitions', 2
+    # it_behaves_like 'a map partitions', nil
+    # it_behaves_like 'a map partitions', rand(2..10)
   end
 
   context 'throught text_file' do
@@ -80,8 +81,9 @@ RSpec::describe 'Spark::RDD.map_partitions(_with_index)' do
       $sc.text_file(file, workers)
     end
 
-    it_behaves_like 'a map partitions', nil
     it_behaves_like 'a map partitions', 1
-    it_behaves_like 'a map partitions', rand(2..10)
+    it_behaves_like 'a map partitions', 2
+    # it_behaves_like 'a map partitions', nil
+    # it_behaves_like 'a map partitions', rand(2..10)
   end
 end

data/spec/lib/map_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a mapping" do |workers|
+RSpec.shared_examples 'a mapping' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers).map(func1)
     result = numbers.map(&func1)
@@ -24,7 +24,7 @@ RSpec::shared_examples "a mapping" do |workers|
   end
 end
 
-RSpec::shared_examples "a mapping values" do |workers|
+RSpec.shared_examples 'a mapping values' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers).map_values(func1)
     result = hash.map{|key, value| [key, func1.call(value)]}
@@ -43,49 +43,52 @@ RSpec::shared_examples "a mapping values" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:func1) { lambda{|x| x*2} }
   let(:func2) { lambda{|x| x*3} }
   let(:func3) { lambda{|x| x*4} }
 
-  context "throught parallelize" do
-    context ".map" do
+  context 'throught parallelize' do
+    context '.map' do
       let(:numbers) { Generator.numbers }
 
       def rdd(workers)
         $sc.parallelize(numbers, workers)
       end
 
-      it_behaves_like "a mapping", nil
-      it_behaves_like "a mapping", 1
-      it_behaves_like "a mapping", rand(2..10)
+      it_behaves_like 'a mapping', 1
+      it_behaves_like 'a mapping', 2
+      # it_behaves_like 'a mapping', nil
+      # it_behaves_like 'a mapping', rand(2..10)
     end
 
-    context ".map_values" do
+    context '.map_values' do
       let!(:hash) { Generator.hash }
 
       def rdd(workers)
         $sc.parallelize(hash, workers)
       end
 
-      it_behaves_like "a mapping values", nil
-      it_behaves_like "a mapping values", 1
-      it_behaves_like "a mapping values", rand(2..10)
+      it_behaves_like 'a mapping values', 1
+      it_behaves_like 'a mapping values', 2
+      # it_behaves_like 'a mapping values', nil
+      # it_behaves_like 'a mapping values', rand(2..10)
     end
   end
 
-  context "throught text_file" do
-    context ".map" do
-      let(:file) { File.join("spec", "inputs", "numbers_0_100.txt") }
+  context 'throught text_file' do
+    context '.map' do
+      let(:file) { File.join('spec', 'inputs', 'numbers_0_100.txt') }
       let(:numbers) { File.readlines(file).map(&:strip) }
 
       def rdd(workers)
         $sc.text_file(file, workers)
       end
 
-      it_behaves_like "a mapping", nil
-      it_behaves_like "a mapping", 1
-      it_behaves_like "a mapping", rand(2..10)
+      it_behaves_like 'a mapping', 1
+      it_behaves_like 'a mapping', 2
+      # it_behaves_like 'a mapping', nil
+      # it_behaves_like 'a mapping', rand(2..10)
     end
   end
 end